I have a shopify store and I am trying to read the data from it using singer.io and dump it into the redshift cluster. I was able to run it successfully using postgresql but redshift is throwing an error that I am unable to understand and fix. Any help or suggestions as to how to get around this?
Below is my redshfit_config.json
{
"redshift_host":"redshift-cluster-1.cepdr3dnmmcm.us-east-1.redshift.amazonaws.com",
"redshift_port":5439,
"redshift_database":"dev",
"redshift_username":"awsuser",
"redshift_password":"Password",
"redshift_schema":"singer",
"default_column_length":1000,
"target_s3":{
"aws_access_key_id":"*Aws key id*",
"aws_secret_access_key":"*aws access key*",
"bucket":"singer-redshift",
"key_prefix":"__tmp"
}
and when trying to run i am getting this error
root@e0e876755adf:/# ~/.virtualenvs/tap-shopify/bin/tap-shopify -c config.json --catalog catalog.json | target-redshift --config redshift_config.json
INFO GET https://testnoman.myshopify.com/admin/api/2024-01/shop.json
Traceback (most recent call last):
File "/usr/local/bin/target-redshift", line 5, in <module>
from target_redshift import cli
File "/usr/local/lib/python3.10/dist-packages/target_redshift/__init__.py", line 8, in <module>
from target_redshift.s3 import S3
File "/usr/local/lib/python3.10/dist-packages/target_redshift/s3.py", line 3, in <module>
import boto3
File "/usr/local/lib/python3.10/dist-packages/boto3/__init__.py", line 16, in <module>
from boto3.session import Session
File "/usr/local/lib/python3.10/dist-packages/boto3/session.py", line 17, in <module>
import botocore.session
File "/usr/local/lib/python3.10/dist-packages/botocore/session.py", line 29, in <module>
import botocore.configloader
File "/usr/local/lib/python3.10/dist-packages/botocore/configloader.py", line 19, in <module>
from botocore.compat import six
File "/usr/local/lib/python3.10/dist-packages/botocore/compat.py", line 25, in <module>
from botocore.exceptions import MD5UnavailableError
File "/usr/local/lib/python3.10/dist-packages/botocore/exceptions.py", line 15, in <module>
from botocore.vendored import requests
File "/usr/local/lib/python3.10/dist-packages/botocore/vendored/requests/__init__.py", line 58, in <module>
from . import utils
File "/usr/local/lib/python3.10/dist-packages/botocore/vendored/requests/utils.py", line 26, in <module>
from .compat import parse_http_list as _parse_list_header
File "/usr/local/lib/python3.10/dist-packages/botocore/vendored/requests/compat.py", line 7, in <module>
from .packages import chardet
File "/usr/local/lib/python3.10/dist-packages/botocore/vendored/requests/packages/__init__.py", line 3, in <module>
from . import urllib3
File "/usr/local/lib/python3.10/dist-packages/botocore/vendored/requests/packages/urllib3/__init__.py", line 10, in <module>
from .connectionpool import (
File "/usr/local/lib/python3.10/dist-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py", line 38, in <module>
from .response import HTTPResponse
File "/usr/local/lib/python3.10/dist-packages/botocore/vendored/requests/packages/urllib3/response.py", line 9, in <module>
from ._collections import HTTPHeaderDict
File "/usr/local/lib/python3.10/dist-packages/botocore/vendored/requests/packages/urllib3/_collections.py", line 1, in <module>
from collections import Mapping, MutableMapping
ImportError: cannot import name 'Mapping' from 'collections' (/usr/lib/python3.10/collections/__init__.py)
INFO --> 200 OK 1517b
CRITICAL BrokenPipeError
Traceback (most recent call last):
File "/root/.virtualenvs/tap-shopify/lib/python3.10/site-packages/tap_shopify/__init__.py", line 213, in main
sync()
File "/root/.virtualenvs/tap-shopify/lib/python3.10/site-packages/tap_shopify/__init__.py", line 149, in sync
singer.write_schema(stream["tap_stream_id"],
File "/root/.virtualenvs/tap-shopify/lib/python3.10/site-packages/singer/messages.py", line 264, in write_schema
write_message(
File "/root/.virtualenvs/tap-shopify/lib/python3.10/site-packages/singer/messages.py", line 227, in write_message
sys.stdout.flush()
BrokenPipeError: [Errno 32] Broken pipe
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/root/.virtualenvs/tap-shopify/bin/tap-shopify", line 33, in <module>
sys.exit(load_entry_point('tap-shopify==1.10.0', 'console_scripts', 'tap-shopify')())
File "/root/.virtualenvs/tap-shopify/lib/python3.10/site-packages/singer/utils.py", line 235, in wrapped
return fnc(*args, **kwargs)
File "/root/.virtualenvs/tap-shopify/lib/python3.10/site-packages/tap_shopify/__init__.py", line 228, in main
raise ShopifyError(exc) from exc
tap_shopify.exceptions.ShopifyError: BrokenPipeError
If I switch the target from redshift to postgres it works fine.
I am running python 3.10