From 570b8d61f0f68fd817011dcac7132ec999ee9594 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 19 Nov 2025 13:07:03 -0800 Subject: [PATCH] improvement(runners): added blacksmith optimizations to workflows and dockerfiles to enhance performance (#2055) * added blacksmith optimizations to workflows and dockerfiles to enhance performance. please review before pushing to production * remove cache from and cache to directives from docker based actions, per blacksmith docs --------- Co-authored-by: Connor Mulholland --- .github/workflows/docs-embeddings.yml | 13 ++++++++++- .github/workflows/i18n.yml | 24 +++++++++++++++++++- .github/workflows/migrations.yml | 13 ++++++++++- .github/workflows/publish-cli.yml | 13 ++++++++++- .github/workflows/publish-ts-sdk.yml | 13 ++++++++++- .github/workflows/test-build.yml | 11 +++++++++ .github/workflows/trigger-deploy.yml | 13 ++++++++++- docker/app.Dockerfile | 28 +++++++++++++++++------ docker/db.Dockerfile | 22 +++++++++++++----- docker/realtime.Dockerfile | 32 ++++++++++++++++++++------- 10 files changed, 155 insertions(+), 27 deletions(-) diff --git a/.github/workflows/docs-embeddings.yml b/.github/workflows/docs-embeddings.yml index 5f51d4b0c..e01024a0e 100644 --- a/.github/workflows/docs-embeddings.yml +++ b/.github/workflows/docs-embeddings.yml @@ -24,8 +24,19 @@ jobs: with: node-version: latest + - name: Cache Bun dependencies + uses: actions/cache@v4 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + - name: Install dependencies - run: bun install + run: bun install --frozen-lockfile - name: Process docs embeddings working-directory: ./apps/sim diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml index d71cdffd6..e98fc6922 100644 --- a/.github/workflows/i18n.yml +++ b/.github/workflows/i18n.yml @@ -28,6 +28,17 @@ jobs: with: bun-version: 1.2.22 + - name: Cache Bun dependencies + uses: actions/cache@v4 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + - name: Run Lingo.dev translations env: LINGODOTDEV_API_KEY: ${{ secrets.LINGODOTDEV_API_KEY }} @@ -117,10 +128,21 @@ jobs: with: bun-version: 1.2.22 + - name: Cache Bun dependencies + uses: actions/cache@v4 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + - name: Install dependencies run: | cd apps/docs - bun install + bun install --frozen-lockfile - name: Build documentation to verify translations run: | diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index 2bfb6ca1c..cc1b471a8 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -18,8 +18,19 @@ jobs: with: bun-version: 1.2.22 + - name: Cache Bun dependencies + uses: actions/cache@v4 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + - name: Install dependencies - run: bun install + run: bun install --frozen-lockfile - name: Apply migrations working-directory: ./packages/db diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.yml index 3e48ac6dc..163a2ecd8 100644 --- a/.github/workflows/publish-cli.yml +++ b/.github/workflows/publish-cli.yml @@ -24,9 +24,20 @@ jobs: node-version: '18' registry-url: 'https://registry.npmjs.org/' + - name: Cache Bun dependencies + uses: actions/cache@v4 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + - name: Install dependencies working-directory: packages/cli - run: bun install + run: bun install --frozen-lockfile - name: Build package working-directory: packages/cli diff --git a/.github/workflows/publish-ts-sdk.yml b/.github/workflows/publish-ts-sdk.yml index 5158705fb..9d8b3b44a 100644 --- a/.github/workflows/publish-ts-sdk.yml +++ b/.github/workflows/publish-ts-sdk.yml @@ -24,8 +24,19 @@ jobs: node-version: '22' registry-url: 'https://registry.npmjs.org/' + - name: Cache Bun dependencies + uses: actions/cache@v4 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + - name: Install dependencies - run: bun install + run: bun install --frozen-lockfile - name: Run tests working-directory: packages/ts-sdk diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 3a7a69f96..fa10b0dd4 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -23,6 +23,17 @@ jobs: with: node-version: latest + - name: Cache Bun dependencies + uses: actions/cache@v4 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + - name: Install dependencies run: bun install --frozen-lockfile diff --git a/.github/workflows/trigger-deploy.yml b/.github/workflows/trigger-deploy.yml index 88a059b28..a3fed7619 100644 --- a/.github/workflows/trigger-deploy.yml +++ b/.github/workflows/trigger-deploy.yml @@ -29,8 +29,19 @@ jobs: with: bun-version: 1.2.22 + - name: Cache Bun dependencies + uses: actions/cache@v4 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + - name: Install dependencies - run: bun install + run: bun install --frozen-lockfile - name: Deploy to Trigger.dev (Staging) if: github.ref == 'refs/heads/staging' diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index 44087f9b6..b55a3953b 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -10,7 +10,7 @@ FROM base AS deps RUN apk add --no-cache libc6-compat WORKDIR /app -# Install turbo globally +# Install turbo globally (cached separately, changes infrequently) RUN bun install -g turbo COPY package.json bun.lock turbo.json ./ @@ -18,6 +18,7 @@ RUN mkdir -p apps packages/db COPY apps/sim/package.json ./apps/sim/package.json COPY packages/db/package.json ./packages/db/package.json +# Install dependencies (this layer will be cached if package files don't change) RUN bun install --omit dev --ignore-scripts # ======================================== @@ -26,14 +27,26 @@ RUN bun install --omit dev --ignore-scripts FROM base AS builder WORKDIR /app -# Install turbo globally in builder stage +# Install turbo globally (cached separately, changes infrequently) RUN bun install -g turbo +# Copy node_modules from deps stage (cached if dependencies don't change) COPY --from=deps /app/node_modules ./node_modules -COPY . . -# Installing with full context to prevent missing dependencies error -RUN bun install --omit dev --ignore-scripts +# Copy package configuration files (needed for build) +COPY package.json bun.lock turbo.json ./ +COPY apps/sim/package.json ./apps/sim/package.json +COPY packages/db/package.json ./packages/db/package.json + +# Copy workspace configuration files (needed for turbo) +COPY apps/sim/next.config.ts ./apps/sim/next.config.ts +COPY apps/sim/tsconfig.json ./apps/sim/tsconfig.json +COPY apps/sim/tailwind.config.ts ./apps/sim/tailwind.config.ts +COPY apps/sim/postcss.config.mjs ./apps/sim/postcss.config.mjs + +# Copy source code (changes most frequently - placed last to maximize cache hits) +COPY apps/sim ./apps/sim +COPY packages ./packages # Required for standalone nextjs build WORKDIR /app/apps/sim @@ -64,15 +77,16 @@ RUN bun run build FROM base AS runner WORKDIR /app -# Install Python and dependencies for guardrails PII detection +# Install Python and dependencies for guardrails PII detection (cached separately) RUN apk add --no-cache python3 py3-pip bash ENV NODE_ENV=production -# Create non-root user and group +# Create non-root user and group (cached separately) RUN addgroup -g 1001 -S nodejs && \ adduser -S nextjs -u 1001 +# Copy application artifacts from builder (these change on every build) COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/public ./apps/sim/public COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/standalone ./ COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/static ./apps/sim/.next/static diff --git a/docker/db.Dockerfile b/docker/db.Dockerfile index 32c8f3add..31366eb36 100644 --- a/docker/db.Dockerfile +++ b/docker/db.Dockerfile @@ -1,29 +1,39 @@ +# ======================================== +# Base Stage: Alpine Linux with Bun +# ======================================== +FROM oven/bun:1.2.22-alpine AS base + # ======================================== # Dependencies Stage: Install Dependencies # ======================================== -FROM oven/bun:1.2.22-alpine AS deps +FROM base AS deps WORKDIR /app -# Copy only package files needed for migrations +# Copy only package files needed for migrations (these change less frequently) COPY package.json bun.lock turbo.json ./ +RUN mkdir -p packages/db COPY packages/db/package.json ./packages/db/package.json -# Install dependencies +# Install dependencies (this layer will be cached if package files don't change) RUN bun install --ignore-scripts # ======================================== # Runner Stage: Production Environment # ======================================== -FROM oven/bun:1.2.22-alpine AS runner +FROM base AS runner WORKDIR /app -# Create non-root user and group +# Create non-root user and group (cached separately) RUN addgroup -g 1001 -S nodejs && \ adduser -S nextjs -u 1001 -# Copy only the necessary files from deps +# Copy only the necessary files from deps (cached if dependencies don't change) COPY --from=deps --chown=nextjs:nodejs /app/node_modules ./node_modules + +# Copy package configuration files (needed for migrations) COPY --chown=nextjs:nodejs packages/db/drizzle.config.ts ./packages/db/drizzle.config.ts + +# Copy database package source code (changes most frequently - placed last) COPY --chown=nextjs:nodejs packages/db ./packages/db # Switch to non-root user diff --git a/docker/realtime.Dockerfile b/docker/realtime.Dockerfile index ebf1ea466..1e2ae6084 100644 --- a/docker/realtime.Dockerfile +++ b/docker/realtime.Dockerfile @@ -10,7 +10,7 @@ FROM base AS deps RUN apk add --no-cache libc6-compat WORKDIR /app -# Install turbo globally +# Install turbo globally (cached separately, changes infrequently) RUN bun install -g turbo COPY package.json bun.lock turbo.json ./ @@ -18,16 +18,26 @@ RUN mkdir -p apps packages/db COPY apps/sim/package.json ./apps/sim/package.json COPY packages/db/package.json ./packages/db/package.json +# Install dependencies (this layer will be cached if package files don't change) RUN bun install --omit dev --ignore-scripts # ======================================== -# Builder Stage: Build the Application +# Builder Stage: Prepare source code # ======================================== FROM base AS builder WORKDIR /app +# Copy node_modules from deps stage (cached if dependencies don't change) COPY --from=deps /app/node_modules ./node_modules -COPY . . + +# Copy package configuration files (needed for build) +COPY package.json bun.lock turbo.json ./ +COPY apps/sim/package.json ./apps/sim/package.json +COPY packages/db/package.json ./packages/db/package.json + +# Copy source code (changes most frequently - placed last to maximize cache hits) +COPY apps/sim ./apps/sim +COPY packages ./packages # ======================================== # Runner Stage: Run the Socket Server @@ -37,16 +47,22 @@ WORKDIR /app ENV NODE_ENV=production -# Create non-root user and group +# Create non-root user and group (cached separately) RUN addgroup -g 1001 -S nodejs && \ adduser -S nextjs -u 1001 -# Copy the sim app and the shared db package needed by socket-server -COPY --from=builder --chown=nextjs:nodejs /app/apps/sim ./apps/sim -COPY --from=builder --chown=nextjs:nodejs /app/packages/db ./packages/db -COPY --from=builder --chown=nextjs:nodejs /app/node_modules ./node_modules +# Copy package.json first (changes less frequently) COPY --from=builder --chown=nextjs:nodejs /app/package.json ./package.json +# Copy node_modules from builder (cached if dependencies don't change) +COPY --from=builder --chown=nextjs:nodejs /app/node_modules ./node_modules + +# Copy db package (needed by socket-server) +COPY --from=builder --chown=nextjs:nodejs /app/packages/db ./packages/db + +# Copy sim app (changes most frequently - placed last) +COPY --from=builder --chown=nextjs:nodejs /app/apps/sim ./apps/sim + # Switch to non-root user USER nextjs