Merge pull request '[PR]: enhance (docker): proper caching and smaller image size' (#10349) from nmkj/calckey:v14-docker into develop

Reviewed-on: https://codeberg.org/calckey/calckey/pulls/10349

.dockerignore

@@ -13,6 +13,9 @@ node_modules
 **/node_modules
 report.*.json
 
+# Rust
+packages/backend/native-utils/target/*
+
 # Cypress
 cypress/screenshots
 cypress/videos
@@ -25,9 +28,6 @@ coverage
 !/.config/example.yml
 !/.config/docker_example.env
 
-#docker dev config
-/dev/docker-compose.yml
-
 # misskey
 built
 db
@@ -47,3 +47,4 @@ packages/backend/assets/instance.css
 # dockerignore custom
 .git
 Dockerfile
+docker-compose.yml

Dockerfile

@@ -1,10 +1,19 @@
 ## Install dev and compilation dependencies, build files
-FROM node:19-alpine as build
+FROM alpine:3.18 as build
 WORKDIR /calckey
 
 # Install compilation dependencies
-RUN apk update
+RUN apk add --no-cache --no-progress git alpine-sdk python3 nodejs-current npm rust cargo vips
-RUN apk add --no-cache --no-progress git alpine-sdk python3 rust cargo vips
+
+# Copy only the cargo dependency-related files first, to cache efficiently
+COPY packages/backend/native-utils/Cargo.toml packages/backend/native-utils/Cargo.toml
+COPY packages/backend/native-utils/Cargo.lock packages/backend/native-utils/Cargo.lock
+COPY packages/backend/native-utils/src/lib.rs packages/backend/native-utils/src/
+COPY packages/backend/native-utils/migration/Cargo.toml packages/backend/native-utils/migration/Cargo.toml
+COPY packages/backend/native-utils/migration/src/lib.rs packages/backend/native-utils/migration/src/
+
+# Install cargo dependencies
+RUN cargo fetch --locked --manifest-path /calckey/packages/backend/native-utils/Cargo.toml
 
 # Copy only the dependency-related files first, to cache efficiently
 COPY package.json pnpm*.yaml ./
@@ -16,27 +25,31 @@ COPY packages/backend/native-utils/package.json packages/backend/native-utils/pa
 COPY packages/backend/native-utils/npm/linux-x64-musl/package.json packages/backend/native-utils/npm/linux-x64-musl/package.json
 COPY packages/backend/native-utils/npm/linux-arm64-musl/package.json packages/backend/native-utils/npm/linux-arm64-musl/package.json
 
-# Configure corepack and pnpm
+# Configure corepack and pnpm, and install dev mode dependencies for compilation
-RUN corepack enable
+RUN corepack enable && corepack prepare pnpm@latest --activate && pnpm i --frozen-lockfile
-RUN corepack prepare pnpm@latest --activate
 
-# Install dev mode dependencies for compilation
+# Copy in the rest of the native-utils rust files
-RUN pnpm i --frozen-lockfile
+COPY packages/backend/native-utils/.cargo packages/backend/native-utils/.cargo
+COPY packages/backend/native-utils/build.rs packages/backend/native-utils/
+COPY packages/backend/native-utils/src packages/backend/native-utils/src/
+COPY packages/backend/native-utils/migration/src packages/backend/native-utils/migration/src/
 
-# Copy in the rest of the files, to compile from TS to JS
+# Compile native-utils
+RUN pnpm run --filter native-utils build
+
+# Copy in the rest of the files to compile
 COPY . ./
-RUN pnpm run build
+RUN env NODE_ENV=production sh -c "pnpm run --filter '!native-utils' build && pnpm run gulp"
 
-# Trim down the dependencies to only the prod deps
+# Trim down the dependencies to only those for production
 RUN pnpm i --prod --frozen-lockfile
 
-
 ## Runtime container
-FROM node:19-alpine
+FROM alpine:3.18
 WORKDIR /calckey
 
 # Install runtime dependencies
-RUN apk add --no-cache --no-progress tini ffmpeg vips-dev zip unzip rust cargo
+RUN apk add --no-cache --no-progress tini ffmpeg vips-dev zip unzip nodejs-current
 
 COPY . ./
 
@@ -52,8 +65,9 @@ COPY --from=build /calckey/built /calckey/built
 COPY --from=build /calckey/packages/backend/built /calckey/packages/backend/built
 COPY --from=build /calckey/packages/backend/assets/instance.css /calckey/packages/backend/assets/instance.css
 COPY --from=build /calckey/packages/backend/native-utils/built /calckey/packages/backend/native-utils/built
-COPY --from=build /calckey/packages/backend/native-utils/target /calckey/packages/backend/native-utils/target
 
-RUN corepack enable
+RUN corepack enable && corepack prepare pnpm@latest --activate
+ENV NODE_ENV=production
+VOLUME "/calckey/files"
 ENTRYPOINT [ "/sbin/tini", "--" ]
 CMD [ "pnpm", "run", "migrateandstart" ]

docker-compose.yml

@@ -19,8 +19,6 @@ services:
     environment:
       NODE_ENV: production
     volumes:
-      - ./.cargo-cache:/root/.cargo
-      - ./.cargo-target:/calckey/packages/backend/native-utils/target
       - ./files:/calckey/files
       - ./.config:/calckey/.config:ro
 

packages/backend/native-utils/migration/Cargo.toml

@@ -10,14 +10,14 @@ path = "src/lib.rs"
 
 [features]
 default = []
-convert = ["dep:native-utils"]
+convert = ["dep:native-utils", "dep:indicatif", "dep:futures"]
 
 [dependencies]
 serde_json = "1.0.96"
 native-utils = { path = "../", optional = true }
-indicatif = { version = "0.17.4", features = ["tokio"] }
+indicatif = { version = "0.17.4", features = ["tokio"], optional = true }
 tokio = { version = "1.28.2", features = ["full"] }
-futures = "0.3.28"
+futures = { version = "0.3.28", optional = true }
 serde_yaml = "0.9.21"
 serde = { version = "1.0.163", features = ["derive"] }
 urlencoding = "2.1.2"

packages/backend/native-utils/package.json

@@ -1,48 +1,50 @@
 {
-  "name": "native-utils",
+	"name": "native-utils",
-  "version": "0.0.0",
+	"version": "0.0.0",
-  "main": "built/index.js",
+	"main": "built/index.js",
-  "types": "built/index.d.ts",
+	"types": "built/index.d.ts",
-  "napi": {
+	"napi": {
-    "name": "native-utils",
+		"name": "native-utils",
-    "triples": {
+		"triples": {
-      "additional": [
+			"additional": [
-        "aarch64-apple-darwin",
+				"aarch64-apple-darwin",
-        "aarch64-linux-android",
+				"aarch64-linux-android",
-        "aarch64-unknown-linux-gnu",
+				"aarch64-unknown-linux-gnu",
-        "aarch64-unknown-linux-musl",
+				"aarch64-unknown-linux-musl",
-        "aarch64-pc-windows-msvc",
+				"aarch64-pc-windows-msvc",
-        "armv7-unknown-linux-gnueabihf",
+				"armv7-unknown-linux-gnueabihf",
-        "x86_64-unknown-linux-musl",
+				"x86_64-unknown-linux-musl",
-        "x86_64-unknown-freebsd",
+				"x86_64-unknown-freebsd",
-        "i686-pc-windows-msvc",
+				"i686-pc-windows-msvc",
-        "armv7-linux-androideabi",
+				"armv7-linux-androideabi",
-        "universal-apple-darwin"
+				"universal-apple-darwin"
-      ]
+			]
-    }
+		}
-  },
+	},
-  "license": "MIT",
+	"license": "MIT",
-  "devDependencies": {
+	"devDependencies": {
-    "@napi-rs/cli": "2.16.1",
+		"@napi-rs/cli": "2.16.1",
-    "ava": "5.1.1"
+		"ava": "5.1.1"
-  },
+	},
-  "ava": {
+	"ava": {
-    "timeout": "3m"
+		"timeout": "3m"
-  },
+	},
-  "engines": {
+	"engines": {
-    "node": ">= 10"
+		"node": ">= 10"
-  },
+	},
-  "scripts": {
+	"scripts": {
-    "artifacts": "napi artifacts",
+		"artifacts": "napi artifacts",
-    "build": "napi build --features napi --platform --release ./built/",
+		"build": "pnpm run build:napi && pnpm run build:migration",
-    "build:debug": "napi build --platform",
+		"build:napi": "napi build --features napi --platform --release ./built/",
-    "prepublishOnly": "napi prepublish -t npm",
+		"build:migration": "cargo build --locked --release --manifest-path ./migration/Cargo.toml && cp -v ./target/release/migration ./built/migration",
-    "test": "pnpm run cargo:test && pnpm run build && ava",
+		"build:debug": "napi build --platform ./built/ && cargo build --manifest-path ./migration/Cargo.toml",
-    "universal": "napi universal",
+		"prepublishOnly": "napi prepublish -t npm",
-    "version": "napi version",
+		"test": "pnpm run cargo:test && pnpm run build:napi && ava",
+		"universal": "napi universal",
+		"version": "napi version",
 		"format": "cargo fmt --all",
 		"cargo:test": "pnpm run cargo:unit && pnpm run cargo:integration",
-    "cargo:unit": "cargo test unit_test && cargo test -F napi unit_test",
+		"cargo:unit": "cargo test unit_test && cargo test -F napi unit_test",
-    "cargo:integration": "cargo test -F noarray int_test -- --test-threads=1"
+		"cargo:integration": "cargo test -F noarray int_test -- --test-threads=1"
-  }
+	}
 }

packages/backend/package.json

@@ -8,10 +8,10 @@
 		"start:test": "NODE_ENV=test pnpm node ./built/index.js",
 		"migrate": "pnpm run migrate:typeorm && pnpm run migrate:cargo",
 		"migrate:typeorm": "typeorm migration:run -d ormconfig.js",
-		"migrate:cargo": "cargo run --manifest-path ./native-utils/migration/Cargo.toml -- up",
+		"migrate:cargo": "./native-utils/built/migration up",
 		"revertmigration": "pnpm run revertmigration:cargo && pnpm run revertmigration:typeorm",
 		"revertmigration:typeorm": "typeorm migration:revert -d ormconfig.js",
-		"revertmigration:cargo": "cargo run --manifest-path ./native-utils/migration/Cargo.toml -- down",
+		"revertmigration:cargo": "./native-utils/built/migration down",
 		"check:connect": "node ./check_connect.js",
 		"build": "pnpm swc src -d built -D",
 		"watch": "pnpm swc src -d built -D -w",
@@ -20,9 +20,6 @@
 		"test": "pnpm run mocha",
 		"format": "pnpm rome format * --write"
 	},
-	"resolutions": {
-		"chokidar": "^3.3.1"
-	},
 	"optionalDependencies": {
 		"@swc/core-android-arm64": "1.3.11",
 		"@tensorflow/tfjs-node": "3.21.1"

scripts/clean-all.js

@@ -1,26 +1,30 @@
-const fs = require('fs');
+const fs = require('node:fs');
 const execa = require('execa');
+const { join } = require('node:path');
 
 (async () => {
-	fs.rmSync(__dirname + '/../packages/backend/built', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/backend/built'), { recursive: true, force: true });
-	fs.rmSync(__dirname + '/../packages/backend/node_modules', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/backend/node_modules'), { recursive: true, force: true });
 
-	fs.rmSync(__dirname + '/../packages/client/built', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/backend/native-utils/built'), { recursive: true, force: true });
-	fs.rmSync(__dirname + '/../packages/client/node_modules', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/backend/native-utils/node_modules'), { recursive: true, force: true });
 
-	fs.rmSync(__dirname + '/../packages/sw/built', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/client/built'), { recursive: true, force: true });
-	fs.rmSync(__dirname + '/../packages/sw/node_modules', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/client/node_modules'), { recursive: true, force: true });
 
-	fs.rmSync(__dirname + '/../built', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/sw/built'), { recursive: true, force: true });
-	fs.rmSync(__dirname + '/../node_modules', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/sw/node_modules'), { recursive: true, force: true });
+
+	fs.rmSync(join(__dirname, '/../built'), { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../node_modules'), { recursive: true, force: true });
 
 	execa('pnpm', ['store', 'prune'], {
-		cwd: __dirname + '/../',
+		cwd: join(__dirname, '/../'),
 		stdio: 'inherit'
 	});
 
 	execa('cargo', ['clean'], {
-		cwd: __dirname + '/../packages/backend/native-utils',
+		cwd: join(__dirname, '/../packages/backend/native-utils'),
 		stdio: 'inherit'
 	});
 })();

scripts/clean.js

@@ -1,8 +1,10 @@
-const fs = require('fs');
+const fs = require('node:fs');
+const { join } = require('node:path');
 
 (async () => {
-	fs.rmSync(__dirname + '/../packages/backend/built', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/backend/built'), { recursive: true, force: true });
-	fs.rmSync(__dirname + '/../packages/client/built', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/backend/native-utils/built'), { recursive: true, force: true });
-	fs.rmSync(__dirname + '/../packages/sw/built', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/client/built'), { recursive: true, force: true });
-	fs.rmSync(__dirname + '/../built', { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../packages/sw/built'), { recursive: true, force: true });
+	fs.rmSync(join(__dirname, '/../built'), { recursive: true, force: true });
 })();