how to send session cookie loaded from a cookielib.MozillaCookieJar compatible file

Question

I have exported cookies from Chrome using cookies.txt Chrome plugin. I can easily replay the request using curl with the cookies exported. Some code like curl -b mycookie.txt -c mycookie.txt $url works perfectly.

However, when I was trying to load the session from mycookie.txt using requests, it can't work. After long time debugging, I fond out that requests will not send session cookies(whose expire values is 0 in cookies file) even though I already loaded the expired cookie with following code:

cj = cookielib.MozillaCookieJar('mycookie.txt')
cj.load(ignore_discard=True, ignore_expires=True)
print cj  #it shows the session cookie already loaded

s = requests.Session()
s.cookies = cj
r = s.get(url, headers=myheaders, timeout=10, allow_redirects=False)
print r.request.headers   #this clearly shows the request didn't send the session cookie

How can I make it work?

Answer 1

It happened to me. Let's take a closer look at requests/models.py:

 def prepare(
    ...
    self.prepare_cookies(cookies) # [1]


 def prepare_cookies(self, cookies):
   ...
    if isinstance(cookies, cookielib.CookieJar):
        self._cookies = cookies
    else:
        ...
    cookie_header = get_cookie_header(self._cookies, self) # [2]
    ...

Goto requests/cookies.py:

 def get_cookie_header(jar, request):
    ...
    jar.add_cookie_header(r) # [3]
    return r.get_new_headers().get("Cookie")

Goto http/cookiejar.py:

def add_cookie_header(self, request):
    ...
    cookies = self._cookies_for_request(request) # [4]
    ...

def _cookies_for_request(self, request):
    """Return a list of cookies to be returned to server."""
    cookies = []
    for domain in self._cookies.keys():
        cookies.extend(self._cookies_for_domain(domain, request)) # [5]
    return cookies

def _cookies_for_domain(self, domain, request):
    cookies = []
    if not self._policy.domain_return_ok(domain, request):
        return []
    _debug("Checking %s for cookies to return", domain)
    cookies_by_path = self._cookies[domain]
    for path in cookies_by_path.keys():
        if not self._policy.path_return_ok(path, request):
            continue
        cookies_by_name = cookies_by_path[path]
        for cookie in cookies_by_name.values():
            if not self._policy.return_ok(cookie, request): # [6] It calls `return_ok()` per cookie line
                _debug("   not returning cookie")
                continue
            cookies.append(cookie)
    return cookies

def return_ok(self, cookie, request):
    ...

    for n in "version", "verifiability", "secure", "expires", "port", "domain":
        fn_name = "return_ok_"+n
        fn = getattr(self, fn_name)
        if not fn(cookie, request): # [7] It will calls `return_ok_domain()`
            return False
    return True

def return_ok_domain(self, cookie, request):
    req_host, erhn = eff_request_host(request) # [8] It calls eff_request_host() to append .local
    domain = cookie.domain

    if domain and not domain.startswith("."):
        dotdomain = "." + domain
    else:
        dotdomain = domain
    ...

    # [10] `.localhost.local` not endwith `.localhost` causes the cookies to be empty later:
    if cookie.version == 0 and not ("."+erhn).endswith(dotdomain):
        _debug("   request-host %s does not match Netscape cookie domain "
               "%s", req_host, domain)
        return False
    return True

IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
def eff_request_host(request):
    """Return a tuple (request-host, effective request-host name).

    As defined by RFC 2965, except both are lowercased.

    """
    erhn = req_host = request_host(request)
    if req_host.find(".") == -1 and not IPV4_RE.search(req_host):
        erhn = req_host + ".local" # [9] Problem here
    return req_host, erhn

It means that localhost without a dot will be concatenated to localhost.local. If, for instance, my cookie file is edited with the localhost domain, it will not send the cookie because the domain does not match (curl has no such issue):

localhost   FALSE   /   FALSE   1712121230  token   xxxxx

The cookie will be sent after it appends .local (Note that http's localhost only selects lines with the fourth column equal to FALSE):

localhost.local FALSE   /   FALSE   1712121230  token   xxxxx

Alternatively, I can use the same IP, e.g., s.get(http://127.0.0.1:<port>/...) and 127.0.0.1 FALSE / FALSE 1712121230 token xxxxx