Python requests: URL base in Session

Question

When using a Session, it seems you need to provide the full URL each time, e.g.

session = requests.Session()
session.get(\'http://myserver/getstuff\')
sessio         


        

Answer 1

This feature has been asked on the forums a few times 1, 2, 3. The preferred approach as documented here, is subclassing, as follows:

from requests import Session
from urlparse import urljoin

class LiveServerSession(Session):
    def __init__(self, prefix_url=None, *args, **kwargs):
        super(LiveServerSession, self).__init__(*args, **kwargs)
        self.prefix_url = prefix_url

    def request(self, method, url, *args, **kwargs):
        url = urljoin(self.prefix_url, url)
        return super(LiveServerSession, self).request(method, url, *args, **kwargs)

You would use this simply as follows:

baseUrl = 'http://api.twitter.com'
with LiveServerSession(baseUrl) as s:
    resp = s.get('/1/statuses/home_timeline.json')

Answer 2

requests_toolbelt.sessions.BaseUrlSession https://github.com/requests/toolbelt/blob/f5c86c51e0a01fbc8b3b4e1c286fd5c7cb3aacfa/requests_toolbelt/sessions.py#L6

NOTE: This uses urljoin from standard lib. Beware of urljoin's behavior.

In [14]: from urlparse import urljoin

In [15]: urljoin('https://localhost/api', '/resource')
Out[15]: 'https://localhost/resource'

In [16]: urljoin('https://localhost/api', 'resource')
Out[16]: 'https://localhost/resource'

In [17]: urljoin('https://localhost/api/', '/resource')
Out[17]: 'https://localhost/resource'

In [18]: urljoin('https://localhost/api/', 'resource')
Out[18]: 'https://localhost/api/resource'

OR

import requests 
from functools import partial

def PrefixUrlSession(prefix=None):                                                                                                                                                                                                                                                                                                                 
     if prefix is None:                                                                                                                                                                                                                                                                                                                             
         prefix = ""                                                                                                                                                                                                                                                                                                                                
     else:                                                                                                                                                                                                                                                                                                                                          
         prefix = prefix.rstrip('/') + '/'                                                                                                                                                                                                                                                                                                          

     def new_request(prefix, f, method, url, *args, **kwargs):                                                                                                                                                                                                                                                                                      
         return f(method, prefix + url, *args, **kwargs)                                                                                                                                                                                                                                                                                            

     s = requests.Session()                                                                                                                                                                                                                                                                                                                         
     s.request = partial(new_request, prefix, s.request)                                                                                                                                                                                                                                                                                            
     return s             

Answer 3

You could just subclass request.Session and overload its __init__ and request methods like this:

# my_requests.py
import requests


class SessionWithUrlBase(requests.Session):
    # In Python 3 you could place `url_base` after `*args`, but not in Python 2.
    def __init__(self, url_base=None, *args, **kwargs):
        super(SessionWithUrlBase, self).__init__(*args, **kwargs)
        self.url_base = url_base

    def request(self, method, url, **kwargs):
        # Next line of code is here for example purposes only.
        # You really shouldn't just use string concatenation here,
        # take a look at urllib.parse.urljoin instead.
        modified_url = self.url_base + url

        return super(SessionWithUrlBase, self).request(method, modified_url, **kwargs)

And then you could use your subclass instead of requests.Session in your code:

from my_requests import SessionWithUrlBase


session = SessionWithUrlBase(url_base='https://stackoverflow.com/')
session.get('documentation')  # https://stackoverflow.com/documentation

Also you could monkey-patch requests.Session to avoid modifying existing codebase (this implementation should be 100% compatible), but be sure to do actual patching before any code calls requests.Session():

# monkey_patch.py
import requests


class SessionWithUrlBase(requests.Session):
    ...

requests.Session = SessionWithUrlBase

And then:

# main.py
import requests
import monkey_patch


session = requests.Session()
repr(session)  # <monkey_patch.SessionWithUrlBase object at ...>

Answer 4

I don't see a built-in way to do this, but you can use wrapper functions to add the functionality you want:

from functools import wraps
import inspect
import requests
from requests.compat import urljoin

def _base_url(func, base):
    '''Decorator for adding a base URL to func's url parameter'''

    @wraps(func)
    def wrapper(*args, **kwargs):
        argname = 'url'
        argspec = inspect.getargspec(func)

        if argname in kwargs:
            kwargs[argname] = urljoin(base, kwargs[argname])
        else:
            # Find and replace url parameter in positional args. The argspec
            # includes self while args doesn't, so indexes have to be shifted
            # over one
            for i, name in enumerate(argspec[0]):
                if name == argname:
                    args = list(args)
                    args[i-1] = urljoin(base, args[i-1])
                    break

        return func(*args, **kwargs)
    return wrapper

def inject_base_url(func):
    '''Decorator for adding a base URL to all methods that take a url param'''

    @wraps(func)
    def wrapper(*args, **kwargs):
        argname = 'base_url'

        if argname in kwargs:
            obj = args[0]

            # Add base_url decorator to all methods that have a url parameter
            for name, method in inspect.getmembers(obj, inspect.ismethod):
                argspec = inspect.getargspec(method.__func__)

                if 'url' in argspec[0]:
                    setattr(obj, name, _base_url(method, kwargs[argname]))

            del kwargs[argname]

        return func(*args, **kwargs)
    return wrapper

# Wrap requests.Session.__init__ so it takes a base_url parameter
setattr(
    requests.Session,
    '__init__',
    inject_base_url(getattr(requests.Session, '__init__'))
)

Now you can specify a base URL when you construct a new requests.Session object:

s = requests.Session(base_url='http://stackoverflow.com')
s.get('questions')      # http://stackoverflow.com/questions
s.post('documentation') # http://stackoverflow.com/documentation

# With no base_url, you get the default behavior
s = requests.Session()
s.get('http://google.com')

Answer 5

keep it simple and use builtin methods for joining (no '/' suffix hassle):

import urllib.parse
session = requests.Session()
session.my_base_url_join = lambda path: urllib.parse.urljoin(str_BASE_URL, path)
# use: session.get(session.my_base_url_join(path='/message'))