I'm a beginner of Cloud Run and k8s. Now I'm developing with Node.js to use Cloud Run and trying to use SecretManager to save secrets. But, it's never successful because the request to Metadata Service on minikube pods failed.
Error:
Error: Could not refresh access token: request to http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token?scopes=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform failed, reason: socket hang up
at Gaxios._request (/Users/takamizawa/dev/functions/node_modules/gaxios/build/src/gaxios.js:148:19)
at process.processTicksAndRejections (/Users/takamizawa/dev/functions/lib/internal/process/task_queues.js:95:5)
at async metadataAccessor (/Users/takamizawa/dev/functions/node_modules/gcp-metadata/build/src/index.js:94:21)
at async Compute.refreshTokenNoCache (/Users/takamizawa/dev/functions/node_modules/google-auth-library/build/src/auth/computeclient.js:57:20)
at async Compute.getRequestMetadataAsync (/Users/takamizawa/dev/functions/node_modules/google-auth-library/build/src/auth/oauth2client.js:298:17)
at async Compute.getRequestHeaders (/workspace/node_modules/google-auth-library/build/src/auth/oauth2client.js:261:26) {config: {…}, response: undefined, error: FetchError, code: 'ECONNRESET', note: 'Exception occurred in retry method that was not classified as transient', …}
The code is able to get the secret from SecretManager when the script is executed on my shell. I think the problem is with the request to the Metadata Service in the pod, since the request outside the pod is successful. Does anyone know how to solve this problem?
Environment: Cloud Code for VS Code on Mac, Ventura(13.6)
launch.json
{
"configurations": [
{
"name": "Cloud Run: Run/Debug Locally",
"type": "cloudcode.cloudrun",
"request": "launch",
"build": {
"buildpacks": {
"path": "PATH_TO/package.json",
"builder": "gcr.io/buildpacks/builder:latest"
}
},
"image": "functions",
"service": {
"name": "functions",
"containerPort": 8080,
"resources": {
"limits": {
"memory": "512Mi"
}
}
},
"target": {
"minikube": {}
},
"watch": true
}
]
}
package.json
{
"name": "My Cloud Run",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "node index.js"
},
"engines": {
"node": ">=16.0.0"
},
"author": "",
"license": "ISC",
"dependencies": {
"@google-cloud/secret-manager": "^5.0.1",
"express": "^4.18.2"
}
}
index.js
const express = require('express');
const bodyParser = require('body-parser');
const crypto = require('crypto');
const {SecretManagerServiceClient} = require('@google-cloud/secret-manager');
const app = express();
app.use(bodyParser.raw({type: 'application/json'}));
app.post('/', async (req, res) => {
console.log('Request received');
try {
const HMAC = req.headers['x-hmac'];
const isValid = await verifyWebhook(HMAC, req.body);
if (!isValid) {
res.status(401).send('Unauthorized');
return;
}
const body = JSON.parse(req.body);
// some process will be implemented.
res.send();
} catch (err) {
console.error(err);
res.status(500).send('Internal Server Error');
}
});
const verifyWebhook = async (HMAC, body) => {
const secret = await getSecret();
const hash = crypto.createHmac('sha256', secret).update(body).digest('base64');
return HMAC === hash;
}
const getSecret = async () => {
const client = new SecretManagerServiceClient({fallback: true});
const name = process.env.SECRET_NAME;
const [secret] = await client.accessSecretVersion({
name: name,
},
{
timeout: 200000
});
return secret.payload.data.toString();
}
const port = parseInt(process.env.PORT) || 8080;
app.listen(port, () => {
console.log(`Listening on port ${port}`);
});
module.exports = app;
Updated
Cloud Run: Run/Debug Locally
Starting to run the app using configuration 'Cloud Run: Run/Debug Locally' from .vscode/launch.json...
To view more detailed logs, go to Output channel : "Cloud Run: Run/Debug Locally - Detailed"
Dependency check started
Dependency check succeeded
Starting minikube, this may take a while..............................
minikube successfully started
The minikube profile 'cloud-run-dev-internal' has been scheduled to stop automatically after exiting Cloud Code. To disable this on future deployments, set autoStop to false in your launch configuration /Users/takamizawa/dev/functions/.vscode/launch.json
Update initiated
Build started for artifact functions
Build completed for artifact functions
Deploy started
Status check started
Resource pod/functions-6cff78bccd-gt2wm status updated to In Progress
Resource deployment/functions status updated to In Progress
Resource pod/functions-6cff78bccd-gt2wm status updated to In Progress
Resource deployment/functions status completed successfully
Status check succeeded
**************URLs*****************
Deploy completed
Forwarded URL from service functions: http://localhost:8080
Update succeeded
***********************************
Watching for changes...
To disable watch mode for subsequent runs, set watch to false in your launch configuration /Users/takamizawa/dev/functions/.vscode/launch.json and relaunch the application.
Cloud Run: Run/Debug Locally - Detailed
Running skaffold command: /usr/local/opt/google-cloud-sdk/bin/skaffold init --artifact {"image":"functions","builder":"Buildpacks","payload":{"path":"focus-on/shopify/kintone/package.json","builder":"gcr.io/buildpacks/builder:latest"}} --kubernetes-manifest /var/folders/h5/plq4qwpx6tv0rcs5dzl2_2tr0000gn/T/cloud-code-cloud-run-m6dIbQ/pods_and_services.yaml --filename /var/folders/h5/plq4qwpx6tv0rcs5dzl2_2tr0000gn/T/cloud-code-cloud-run-m6dIbQ/skaffold.yaml --force
Setting build tagPolicy to SHA256.
Setting statusCheckDeadlineSeconds to 5 minutes
Setting build environment to 'local'
Created the file /var/folders/h5/plq4qwpx6tv0rcs5dzl2_2tr0000gn/T/cloud-code-cloud-run-m6dIbQ/skaffold.yaml.
Running gcloud command: /usr/local/opt/google-cloud-sdk/bin/gcloud alpha code export --readiness-probe --service-name functions --image functions --no-skaffold-file --dockerfile= --kubernetes-file /var/folders/h5/plq4qwpx6tv0rcs5dzl2_2tr0000gn/T/cloud-code-cloud-run-m6dIbQ/pods_and_services.yaml --env-vars K_SERVICE=functions,K_REVISION=local,K_CONFIGURATION=functions --memory 512Mi --quiet --verbosity warning
Created the file /var/folders/h5/plq4qwpx6tv0rcs5dzl2_2tr0000gn/T/cloud-code-cloud-run-m6dIbQ/pods_and_services.yaml.
Running: skaffold dev -v info --port-forward --rpc-http-port 50988 --filename /var/folders/h5/plq4qwpx6tv0rcs5dzl2_2tr0000gn/T/cloud-code-cloud-run-m6dIbQ/skaffold.yaml --kube-context cloud-run-dev-internal --minikube-profile cloud-run-dev-internal --status-check\=true
Activated kube-context \"cloud-run-dev-internal\"
starting gRPC server on port 51058
starting gRPC HTTP server on port 50988 (proxying to 51058)
Skaffold &{Version:v2.7.0 ConfigVersion:skaffold/v4beta6 GitVersion: GitCommit:a66f3fa441d2f5ad34db81bb65cc6dc9da818614 BuildDate:2023-08-31T14:37:16Z GoVersion:go1.21.0 Compiler:gc Platform:darwin/amd64 User:}
Loaded Skaffold defaults from \"/Users/takamizawa/.skaffold/config\"
map entry found when executing locate for &{functions focus-on/shopify/kintone 0xc00028e2a0 {<nil> <nil> <nil> <nil> <nil> 0xc000c92120 <nil>} [] {[] []} [] } of type *latest.Artifact and pointer: 824647049424
Using kubectl context: cloud-run-dev-internal
Using minikube docker daemon at tcp://127.0.0.1:50999
no kpt renderer or deployer found, skipping hydrated-dir creation
build concurrency first set to 1 parsed from *local.Builder[0]
final build concurrency value is 1
Generating tags...
- functions -> functions:latest
Checking cache...
Tags generated in 1.216116ms
- functions: Not found. Building
Cache check completed in 14.702 seconds
Starting build...
Found [cloud-run-dev-internal] context, using local docker daemon.
Building [functions]...
Target platforms: [linux/amd64]
latest: Pulling from buildpacks/builder
Digest: sha256:e2e73204454586e414d35c698e5d41607a6dd10c0024c81c47db60320ea2c99f
Status: Image is up to date for gcr.io/buildpacks/builder:latest
latest: Pulling from buildpacks/google-22/run
Digest: sha256:8035d072a47e907f059c21d4f42ab53c804c9f05ccd09a7aac3b431d8059b37c
Status: Image is up to date for gcr.io/buildpacks/google-22/run:latest
0.17.0: Pulling from buildpacksio/lifecycle
Digest: sha256:bc0d791f89bda0e8fbf9bf3e47c2faf1ac4eca4b16f0b01c6cb606883759766a
Status: Image is up to date for buildpacksio/lifecycle:0.17.0
===> ANALYZING
[analyzer] Timer: Analyzer started at 2023-09-28T03:03:34Z
[analyzer] Timer: Analyzer ran for 644.593µs and ended at 2023-09-28T03:03:34Z
===> DETECTING
[detector] Timer: Detector started at 2023-09-28T03:03:38Z
[detector] 3 of 5 buildpacks participating
[detector] google.nodejs.runtime 1.0.0
[detector] google.nodejs.npm 1.0.0
[detector] google.utils.label-image 0.0.2
[detector] Timer: Detector ran for 3.043965996s and ended at 2023-09-28T03:03:42Z
===> RESTORING
[restorer] Timer: Restorer started at 2023-09-28T03:03:43Z
[restorer] Restoring metadata for "google.nodejs.runtime:node" from app image
[restorer] Restoring metadata for "google.nodejs.npm:devmode_scripts" from app image
[restorer] Restoring metadata for "google.nodejs.npm:npm_modules" from cache
[restorer] Removing "google.nodejs.runtime:node", wrong sha
[restorer] Restoring data for "google.nodejs.npm:npm_modules" from cache
[restorer] Timer: Restorer ran for 1.010129113s and ended at 2023-09-28T03:03:44Z
===> BUILDING
[builder] Timer: Builder started at 2023-09-28T03:03:46Z
[builder] === Node.js - Runtime ([email protected]) ===
[builder] 2023/09/28 03:03:46 [DEBUG] GET https://dl.google.com/runtimes/ubuntu2204/nodejs/version.json
[builder] ***** CACHE MISS: "nodejs"
[builder] Installing Node.js v20.7.0.
[builder] 2023/09/28 03:03:47 [DEBUG] GET https://dl.google.com/runtimes/ubuntu2204/nodejs/nodejs-20.7.0.tar.gz
[builder] [33;1mWarning: [0mBOM table is deprecated in this buildpack api version, though it remains supported for backwards compatibility. Buildpack authors should write BOM information to <layer>.sbom.<ext>, launch.sbom.<ext>, or build.sbom.<ext>.
[builder] [33;1mWarning: [0mBOM table is deprecated in this buildpack api version, though it remains supported for backwards compatibility. Buildpack authors should write BOM information to <layer>.sbom.<ext>, launch.sbom.<ext>, or build.sbom.<ext>.
[builder] [33;1mWarning: [0mBOM table is deprecated in this buildpack api version, though it remains supported for backwards compatibility. Buildpack authors should write BOM information to <layer>.sbom.<ext>, launch.sbom.<ext>, or build.sbom.<ext>.
[builder] === Node.js - Npm ([email protected]) ===
[builder] ***** CACHE HIT: "npm_modules"
[builder] --------------------------------------------------------------------------------
[builder] Running "npm install --quiet (NODE_ENV=production)"
[builder]
[builder] up to date, audited 134 packages in 3s
[builder]
[builder] 12 packages are looking for funding
[builder] run `npm fund` for details
[builder]
[builder] found 0 vulnerabilities
[builder] Done "npm install --quiet (NODE_ENV=production)" (3.884987475s)
[builder] ***** CACHE MISS: "watchexec"
[builder] Installing watchexec v1.12.0
[builder] --------------------------------------------------------------------------------
[builder] Running "bash -c curl --fail --show-error --silent --location --retry 3 https://github.com/watchexec/watchexec/releases/download/1.12.0/watchexec-1.12.0-x86_64-unknown-linux-gnu.tar.xz | tar xJ --directory /layers/google.nodejs.npm/watchexec/bin --strip-components=1 --wildcards \"*watchexec\""
[builder] Done "bash -c curl --fail --show-error --silent --location --retry..." (3.089790791s)
[builder] [33;1mWarning: [0mBOM table is deprecated in this buildpack api version, though it remains supported for backwards compatibility. Buildpack authors should write BOM information to <layer>.sbom.<ext>, launch.sbom.<ext>, or build.sbom.<ext>.
[builder] [33;1mWarning: [0mBOM table is deprecated in this buildpack api version, though it remains supported for backwards compatibility. Buildpack authors should write BOM information to <layer>.sbom.<ext>, launch.sbom.<ext>, or build.sbom.<ext>.
[builder] [33;1mWarning: [0mBOM table is deprecated in this buildpack api version, though it remains supported for backwards compatibility. Buildpack authors should write BOM information to <layer>.sbom.<ext>, launch.sbom.<ext>, or build.sbom.<ext>.
[builder] === Utils - Label Image ([email protected]) ===
[builder] Timer: Builder ran for 19.693979233s and ended at 2023-09-28T03:04:06Z
===> EXPORTING
[exporter] Timer: Exporter started at 2023-09-28T03:04:08Z
[exporter] Adding layer 'google.nodejs.runtime:node'
[exporter] Adding layer 'google.nodejs.npm:devmode_scripts'
[exporter] Adding layer 'google.nodejs.npm:env'
[exporter] Adding layer 'google.nodejs.npm:watchexec'
[exporter] Adding 1/1 app layer(s)
[exporter] Reusing layer 'buildpacksio/lifecycle:launcher'
[exporter] Reusing layer 'buildpacksio/lifecycle:config'
[exporter] Reusing layer 'buildpacksio/lifecycle:process-types'
[exporter] Adding label 'io.buildpacks.lifecycle.metadata'
[exporter] Adding label 'io.buildpacks.build.metadata'
[exporter] Adding label 'io.buildpacks.project.metadata'
[exporter] Setting default process type 'web'
[exporter] Timer: Saving functions:latest... started at 2023-09-28T03:04:54Z
[exporter] *** Images (8b5fd2a0ea19):
[exporter] functions:latest
[exporter] Timer: Saving functions:latest... ran for 19.077410554s and ended at 2023-09-28T03:05:13Z
[exporter] Timer: Exporter ran for 1m5.344426136s and ended at 2023-09-28T03:05:13Z
[exporter] Timer: Cache started at 2023-09-28T03:05:13Z
[exporter] Adding cache layer 'google.nodejs.runtime:node'
[exporter] Adding cache layer 'google.nodejs.npm:npm_modules'
[exporter] Adding cache layer 'google.nodejs.npm:watchexec'
[exporter] Timer: Cache ran for 3.090812388s and ended at 2023-09-28T03:05:16Z
Build [functions] succeeded
Build completed in 1 minute 58.473 seconds
Starting render...
starting render process
Render completed in 77.071018ms
Tags used in deployment:
- functions -> functions:8b5fd2a0ea19338e696f5dbb81a77d138ad1aa8c1f7b5d55730b26ddbca43dd8
Starting deploy...
- Warning: spec.template.spec.containers[0].env[4].name: duplicate name "K_CONFIGURATION"
- Warning: spec.template.spec.containers[0].env[5].name: duplicate name "K_REVISION"
- Warning: spec.template.spec.containers[0].env[6].name: duplicate name "K_SERVICE"
- deployment.apps/functions created
- service/functions created
Waiting for deployments to stabilize...
- deployment/functions: creating container functions-container
- pod/functions-6cff78bccd-gt2wm: creating container functions-container
- deployment/functions is ready.
Deployments stabilized in 6.436 seconds
Deploy completed in 8.834 seconds
Port forwarding service/functions in namespace default, remote port 8080 -> http://127.0.0.1:8080
Listing files to watch...
- functions
Streaming logs from pod: functions-6cff78bccd-gt2wm container: functions-readiness-probe
Streaming logs from pod: functions-6cff78bccd-gt2wm container: functions-container
List generated in 36.132169ms
Press Ctrl+C to exit
Watching for changes...
[functions-container]
[functions-container]> [email protected] start
[functions-container]> node index.js
[functions-container]
[functions-container]Listening on port 8080