JSON String processing injects quotes

Question

I get json encoded secrets from a Vaulwarden-Instance by their UUID. I want so loop over any uuid and set environment variables accordingly.

For every field there should be variable (see below)

uuid_USERNAME=username
uuid_PASSWORD=password
uuid_FIELD_fieldname=fieldvalue
...

echo does inject single quotes so that the script breaks in the for loop ehich sets these field-based environment variables.

Can anyone give me a hint on how to handle this situation?

Best regards!

Here is the script:

#!/bin/bash
for item in 55f8864a-ec88-410c-ab0e-8bee2119042a
do
  entry='{"passwordHistory":[{"lastUsedDate":"2024-02-23T17:10:59.343Z","password":"versteckt_name-leerzeichen: versteckt_wert leerzeichen"},{"lastUsedDate":"2024-02-23T17:10:59.343Z","password":"TestPasswort"},{"lastUsedDate":"2024-02-23T16:45:56.662Z","password":"versteckt_name leerzeichen: versteckt_wert leerzeichen"},{"lastUsedDate":"2024-02-23T16:28:04.951Z","password":"versteckt_name: versteckt_wert"}],"revisionDate":"2024-02-23T17:14:48.810Z","creationDate":"2024-02-18T21:04:02.811Z","deletedDate":null,"object":"item","id":"55f8864a-ec88-410c-ab0e-8bee2119042a","organizationId":null,"folderId":null,"type":1,"reprompt":0,"name":"TestName","notes":"notizen notizen1 notizen2","favorite":false,"fields":[{"name":"benutzerdefiniert_name leerzeichen","value":"benutzerdefiniert_wert leerzeichen","type":0,"linkedId":null},{"name":"versteckt_name leerzeichen","value":"versteckt_wert leerzeichen","type":1,"linkedId":null},{"name":"bool_name leerzeichen","value":"false","type":2,"linkedId":null}],"login":{"fido2Credentials":[],"uris":[],"username":"TestBenutzername leerzeichen","password":"TestPasswort leerzeichen","totp":null,"passwordRevisionDate":"2024-02-23T17:10:59.343Z"},"collectionIds":[]}'
  echo $entry
  echo ${item}_USERNAME=$(echo ${entry} | jq -r ".login.username")  > GITHUB_ENV
  echo ${item}_PASSWORD=$(echo $entry | jq -r ".login.password")  >> GITHUB_ENV
  echo ${item}_NOTES=$(echo $entry | jq -r ".notes | select( . != null )") >> GITHUB_ENV
  for field in $(echo $entry | jq -cr .fields[])
  do
    echo $field
    field_name=$(echo $field | jq -r ".name" | sed -e "s/ /_/g")
    field_value=$(echo $field | jq -r ".value")
    echo ${item}_FIELD_${field_name}=${field_value} >> GITHUB_ENV
  done
done

Here is the JSON in pretty:

{
  "passwordHistory": [
    {
      "lastUsedDate": "2024-02-23T17:10:59.343Z",
      "password": "versteckt_name-leerzeichen: versteckt_wert leerzeichen"
    },
    {
      "lastUsedDate": "2024-02-23T17:10:59.343Z",
      "password": "TestPasswort"
    },
    {
      "lastUsedDate": "2024-02-23T16:45:56.662Z",
      "password": "versteckt_name leerzeichen: versteckt_wert leerzeichen"
    },
    {
      "lastUsedDate": "2024-02-23T16:28:04.951Z",
      "password": "versteckt_name: versteckt_wert"
    }
  ],
  "revisionDate": "2024-02-23T17:14:48.810Z",
  "creationDate": "2024-02-18T21:04:02.811Z",
  "deletedDate": null,
  "object": "item",
  "id": "55f8864a-ec88-410c-ab0e-8bee2119042a",
  "organizationId": null,
  "folderId": null,
  "type": 1,
  "reprompt": 0,
  "name": "TestName",
  "notes": "notizen notizen1 notizen2",
  "favorite": false,
  "fields": [
    {
      "name": "benutzerdefiniert_name leerzeichen",
      "value": "benutzerdefiniert_wert leerzeichen",
      "type": 0,
      "linkedId": null
    },
    {
      "name": "versteckt_name leerzeichen",
      "value": "versteckt_wert leerzeichen",
      "type": 1,
      "linkedId": null
    },
    {
      "name": "bool_name leerzeichen",
      "value": "false",
      "type": 2,
      "linkedId": null
    }
  ],
  "login": {
    "fido2Credentials": [],
    "uris": [],
    "username": "TestBenutzername leerzeichen",
    "password": "TestPasswort leerzeichen",
    "totp": null,
    "passwordRevisionDate": "2024-02-23T17:10:59.343Z"
  },
  "collectionIds": []
}

Here is the output:

55f8864a-ec88-410c-ab0e-8bee2119042a_USERNAME=TestBenutzername leerzeichen
55f8864a-ec88-410c-ab0e-8bee2119042a_PASSWORD=TestPasswort leerzeichen
55f8864a-ec88-410c-ab0e-8bee2119042a_NOTES=notizen notizen1 notizen2
{"name":"benutzerdefiniert_name
jq: parse error: Unfinished string at EOF at line 2, column 0
jq: parse error: Unfinished string at EOF at line 2, column 0
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_=
leerzeichen","value":"benutzerdefiniert_wert
jq: parse error: Invalid numeric literal at line 1, column 12
jq: parse error: Invalid numeric literal at line 1, column 12
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_=
leerzeichen","type":0,"linkedId":null}
jq: parse error: Invalid numeric literal at line 1, column 12
jq: parse error: Invalid numeric literal at line 1, column 12
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_=
{"name":"versteckt_name
jq: parse error: Unfinished string at EOF at line 2, column 0
jq: parse error: Unfinished string at EOF at line 2, column 0
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_=
leerzeichen","value":"versteckt_wert
jq: parse error: Invalid numeric literal at line 1, column 12
jq: parse error: Invalid numeric literal at line 1, column 12
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_=
leerzeichen","type":1,"linkedId":null}
jq: parse error: Invalid numeric literal at line 1, column 12
jq: parse error: Invalid numeric literal at line 1, column 12
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_=
{"name":"bool_name
jq: parse error: Unfinished string at EOF at line 2, column 0
jq: parse error: Unfinished string at EOF at line 2, column 0
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_=
leerzeichen","value":"false","type":2,"linkedId":null}
jq: parse error: Invalid numeric literal at line 1, column 12
jq: parse error: Invalid numeric literal at line 1, column 12
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_=

I tried executing this script and tried different methods e.g. echo <<< $entry or piping the output into files and then cat-ing them which didn't do the trick.

The variables should be set like mentioned in the text above.

Answer 1

The issue that you are asking about is that the output of $(echo $entry | jq -cr .fields[]) is being split on spaces, tabs and line feeds when passed to for (assuming the IFS env var has its default value). You want it to be split on line feeds only.

An easy way to fix that is to switch to using read.

Other changes:

• Fixed a slew of injection bugs. (Still assumes that all variables names are valid.)
• jq -cr '.fields[]' is wrong since you do want JSON. (Thankfully, the incorrect -r was simply ignored.)
• Different echo do different things. Best to avoid it in scripts to avoid surprises.
• Replaced sed -e "s/ /_/g" with the use of gsub( " "; "_" ) in the jq program.
• Replaced select( . != null ) with the shorter values.

Not fixed:

• The value of $item contains dashes, an unsuitable character for variable names.

#!/usr/bin/bash

# to_shell_lit() - Creates a shell literal
# Usage: shell_lit="$( to_shell_lit "..." )"
to_shell_lit() {
   printf \'
   printf %s "$1" | sed "s/'/'\\\\''/g"
   printf \'
}

output_var() {
   printf '%s=%s\n' "$1" "$( to_shell_lit "$2" )"
}

exec >GITHUB_ENV

for item in 55f8864a-ec88-410c-ab0e-8bee2119042a; do
   entry='{"passwordHistory":[{"lastUsedDate":"2024-02-23T17:10:59.343Z","password":"versteckt_name-leerzeichen: versteckt_wert leerzeichen"},{"lastUsedDate":"2024-02-23T17:10:59.343Z","password":"TestPasswort"},{"lastUsedDate":"2024-02-23T16:45:56.662Z","password":"versteckt_name leerzeichen: versteckt_wert leerzeichen"},{"lastUsedDate":"2024-02-23T16:28:04.951Z","password":"versteckt_name: versteckt_wert"}],"revisionDate":"2024-02-23T17:14:48.810Z","creationDate":"2024-02-18T21:04:02.811Z","deletedDate":null,"object":"item","id":"55f8864a-ec88-410c-ab0e-8bee2119042a","organizationId":null,"folderId":null,"type":1,"reprompt":0,"name":"TestName","notes":"notizen notizen1 notizen2","favorite":false,"fields":[{"name":"benutzerdefiniert_name leerzeichen","value":"benutzerdefiniert_wert leerzeichen","type":0,"linkedId":null},{"name":"versteckt_name leerzeichen","value":"versteckt_wert leerzeichen","type":1,"linkedId":null},{"name":"bool_name leerzeichen","value":"false","type":2,"linkedId":null}],"login":{"fido2Credentials":[],"uris":[],"username":"TestBenutzername leerzeichen","password":"TestPasswort leerzeichen","totp":null,"passwordRevisionDate":"2024-02-23T17:10:59.343Z"},"collectionIds":[]}'

   output_var "${item}_USERNAME" "$( printf %s "$entry" | jq -r '.login.username' )"
   output_var "${item}_PASSWORD" "$( printf %s "$entry" | jq -r '.login.password' )"
   output_var "${item}_NOTES"    "$( printf %s "$entry" | jq -r '.notes | values' )"

   printf %s "$entry" | jq -c '.fields[]' | while read -r field; do
     field_name=$(  printf %s "$field" | jq -r '.name | gsub( " "; "_" )' )
     field_value=$( printf %s "$field" | jq -r '.value' )
     output_var "${item}_FIELD_${field_name}" "$field_value"
   done
done

Output:

55f8864a-ec88-410c-ab0e-8bee2119042a_USERNAME='TestBenutzername leerzeichen'
55f8864a-ec88-410c-ab0e-8bee2119042a_PASSWORD='TestPasswort leerzeichen'
55f8864a-ec88-410c-ab0e-8bee2119042a_NOTES='notizen notizen1 notizen2'
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_benutzerdefiniert_name_leerzeichen='benutzerdefiniert_wert leerzeichen'
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_versteckt_name_leerzeichen='versteckt_wert leerzeichen'
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_bool_name_leerzeichen='false'

---

Better yet, just do everything in jq.

#!/usr/bin/bash

exec >GITHUB_ENV

for item in 55f8864a-ec88-410c-ab0e-8bee2119042a; do
   entry='{"passwordHistory":[{"lastUsedDate":"2024-02-23T17:10:59.343Z","password":"versteckt_name-leerzeichen: versteckt_wert leerzeichen"},{"lastUsedDate":"2024-02-23T17:10:59.343Z","password":"TestPasswort"},{"lastUsedDate":"2024-02-23T16:45:56.662Z","password":"versteckt_name leerzeichen: versteckt_wert leerzeichen"},{"lastUsedDate":"2024-02-23T16:28:04.951Z","password":"versteckt_name: versteckt_wert"}],"revisionDate":"2024-02-23T17:14:48.810Z","creationDate":"2024-02-18T21:04:02.811Z","deletedDate":null,"object":"item","id":"55f8864a-ec88-410c-ab0e-8bee2119042a","organizationId":null,"folderId":null,"type":1,"reprompt":0,"name":"TestName","notes":"notizen notizen1 notizen2","favorite":false,"fields":[{"name":"benutzerdefiniert_name leerzeichen","value":"benutzerdefiniert_wert leerzeichen","type":0,"linkedId":null},{"name":"versteckt_name leerzeichen","value":"versteckt_wert leerzeichen","type":1,"linkedId":null},{"name":"bool_name leerzeichen","value":"false","type":2,"linkedId":null}],"login":{"fido2Credentials":[],"uris":[],"username":"TestBenutzername leerzeichen","password":"TestPasswort leerzeichen","totp":null,"passwordRevisionDate":"2024-02-23T17:10:59.343Z"},"collectionIds":[]}'

   printf %s "$entry" | jq -r --arg item "$item" '
      "\( $item )_USERNAME=\( .login.username | @sh )",
      "\( $item )_PASSWORD=\( .login.password | @sh )",
      "\( $item )_NOTES=\(    .notes // ""    | @sh )",
      (
         .fields[] |
         ( .name | gsub( " "; "_" ) ) as $field_name  |
         ( .value                   ) as $field_value |
         "\( $item )_FIELD_\( $field_name )=\( $field_value | @sh )"
      )
   '
done

Output:

55f8864a-ec88-410c-ab0e-8bee2119042a_USERNAME='TestBenutzername leerzeichen'
55f8864a-ec88-410c-ab0e-8bee2119042a_PASSWORD='TestPasswort leerzeichen'
55f8864a-ec88-410c-ab0e-8bee2119042a_NOTES='notizen notizen1 notizen2'
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_benutzerdefiniert_name_leerzeichen='benutzerdefiniert_wert leerzeichen'
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_versteckt_name_leerzeichen='versteckt_wert leerzeichen'
55f8864a-ec88-410c-ab0e-8bee2119042a_FIELD_bool_name_leerzeichen='false'