How to use the pycurl.HEADERFUNCTION function in pycurl

mytest.update_context_before(my_context)
        

        # Pre-run initialization of object, generate executable test objects
        templated_test = mytest.realize(my_context)
        result = TestResponse()
        result.test = templated_test
        result.passed = None

        # Request setup
        curl = templated_test.configure_curl(
            timeout=testset_config.timeout, context=my_context, curl_handle=curl_handle)
        headers = MyIO()
        body = MyIO()
        curl.setopt(pycurl.WRITEFUNCTION, body.write)
        curl.setopt(pycurl.HEADERFUNCTION, headers.write)
        if testset_config.verbose:
            curl.setopt(pycurl.VERBOSE, True)
        if testset_config.ssl_insecure:
            curl.setopt(pycurl.SSL_VERIFYPEER, 0)
            curl.setopt(pycurl.SSL_VERIFYHOST, 0)        

        # Pre-request work, wait for input or add a delay before the request runs
        if testset_config.interactive:
            callbacks.log_status("===================================")
            callbacks.log_status("%s" % mytest.name)
            callbacks.log_status("-----------------------------------")
            callbacks.log_status("REQUEST:")
            callbacks.log_status("%s %s" % (templated_test.method, templated_test.url))
            callbacks.log_status("HEADERS:")
            callbacks.log_status("%s" % (templated_test.headers))
            if mytest.body is not None:

self.set_option(pycurl.MAXREDIRS, 5)
        self.set_option(pycurl.NOSIGNAL, 1)
        # Setting this option with even a nonexistent file makes libcurl
        # handle cookie capture and playback automatically.
        self.set_option(pycurl.COOKIEFILE, "/dev/null")
        # Set timeouts to avoid hanging too long
        self.set_timeout(30)
        # Use password identification from .netrc automatically
        self.set_option(pycurl.NETRC, 1)
        # Set up a callback to capture the payload
        def payload_callback(x):
            self.payload += x
        self.set_option(pycurl.WRITEFUNCTION, payload_callback)
        def header_callback(x):
            self.hdr += x
        self.set_option(pycurl.HEADERFUNCTION, header_callback)

fp = pycurl.Curl()
    fp.setopt(pycurl.SSL_VERIFYPEER, 0)
    fp.setopt(pycurl.SSL_VERIFYHOST, 0)
    fp.setopt(pycurl.HEADER, 1)
    fp.setopt(pycurl.USERAGENT, 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0')
    fp.setopt(pycurl.NOSIGNAL, 1)
    fp.setopt(pycurl.FOLLOWLOCATION, int(follow))
    fp.setopt(pycurl.MAXREDIRS, int(max_follow))
    fp.setopt(pycurl.CONNECTTIMEOUT, int(timeout_tcp))
    fp.setopt(pycurl.TIMEOUT, int(timeout))
    fp.setopt(pycurl.PROXY, proxy)
    fp.setopt(pycurl.PROXYTYPE, proxy_type)
    fp.setopt(pycurl.RESOLVE, [resolve])

    headers_output, body_output = StringIO(), StringIO()
    fp.setopt(pycurl.HEADERFUNCTION, headers_output.write)
    fp.setopt(pycurl.HEADER, 0)
    fp.setopt(pycurl.WRITEFUNCTION, body_output.write)

    def debug_func(t, s):
        if max_mem > 0 and request.tell() > max_mem:
            return 0
        if t in (pycurl.INFOTYPE_HEADER_OUT, pycurl.INFOTYPE_DATA_OUT):
            request.write(s)

    max_mem = int(max_mem)
    request = StringIO()
    fp.setopt(pycurl.DEBUGFUNCTION, debug_func)

    fp.setopt(pycurl.VERBOSE, 1)

    if user_pass:

request.headers["Expect"] = ""

    # libcurl adds Pragma: no-cache by default; disable that too
    if "Pragma" not in request.headers:
        request.headers["Pragma"] = ""

    # Request headers may be either a regular dict or HTTPHeaders object
    if isinstance(request.headers, httputil.HTTPHeaders):
        curl.setopt(pycurl.HTTPHEADER,
                    [native_str("%s: %s" % i) for i in request.headers.get_all()])
    else:
        curl.setopt(pycurl.HTTPHEADER,
                    [native_str("%s: %s" % i) for i in request.headers.items()])

    if request.header_callback:
        curl.setopt(pycurl.HEADERFUNCTION,
                    lambda line: request.header_callback(native_str(line)))
    else:
        curl.setopt(pycurl.HEADERFUNCTION,
                    lambda line: _curl_header_callback(headers,
                                                       native_str(line)))
    if request.streaming_callback:
        write_function = request.streaming_callback
    else:
        write_function = buffer.write
    if bytes_type is str:  # py2
        curl.setopt(pycurl.WRITEFUNCTION, write_function)
    else:  # py3
        # Upstream pycurl doesn't support py3, but ubuntu 12.10 includes
        # a fork/port.  That version has a bug in which it passes unicode
        # strings instead of bytes to the WRITEFUNCTION.  This means that
        # if you use a WRITEFUNCTION (which tornado always does), you cannot

def download(url, destination,
            follow_redirects=False, request_timeout=600):
    _error = _effective_url = None
    with open(destination, 'wb') as destination_file:
        hdr = cStringIO.StringIO()
        c = pycurl.Curl()
        c.setopt(pycurl.URL, str(url))
        c.setopt(pycurl.FOLLOWLOCATION, follow_redirects)
        c.setopt(pycurl.HEADERFUNCTION, hdr.write)
        c.setopt(pycurl.WRITEFUNCTION, destination_file.write)
        #c.setopt(pycurl.WRITEFUNCTION, functools.partial(slow_writer, destination_file))
        c.setopt(pycurl.TIMEOUT_MS, int(1000 * request_timeout))
        c.perform()
        code = c.getinfo(pycurl.HTTP_CODE)
        _effective_url = c.getinfo(pycurl.EFFECTIVE_URL)
        if _effective_url == url:
            _effective_url = None
        code = c.getinfo(pycurl.HTTP_CODE)
        if code != 200:
            status_line = hdr.getvalue().splitlines()[0]
            for each in re.findall(r'HTTP\/\S*\s*\d+\s*(.*?)\s*$', status_line):
                _error = each

    response = {'code': code}
    if _error:

def to_pycurl_object(c, req):

        c.setopt(pycurl.MAXREDIRS, 5)

        c.setopt(pycurl.WRITEFUNCTION, req.body_callback)
        c.setopt(pycurl.HEADERFUNCTION, req.header_callback)

        c.setopt(pycurl.NOSIGNAL, 1)
        c.setopt(pycurl.SSL_VERIFYPEER, False)
        c.setopt(pycurl.SSL_VERIFYHOST, 0)

        if PYCURL_PATH_AS_IS:
            c.setopt(pycurl.PATH_AS_IS, 1)

        c.setopt(pycurl.URL, python2_3_convert_to_unicode(req.completeUrl))

        if req.getConnTimeout():
            c.setopt(pycurl.CONNECTTIMEOUT, req.getConnTimeout())

        if req.getTotalTimeout():
            c.setopt(pycurl.TIMEOUT, req.getTotalTimeout())

curl.setopt(pycurl.URL, str(url))

    if username and password:
        curl.setopt(pycurl.USERPWD, "%s:%s" % (str(username), str(password)))

    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.MAXREDIRS, 5)
    curl.setopt(pycurl.TIMEOUT, 15)
    curl.setopt(pycurl.CONNECTTIMEOUT, 8)
    curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_1_0)

    content = StringIO.StringIO()
    hdr = StringIO.StringIO()
    curl.setopt(pycurl.WRITEFUNCTION, content.write)
    curl.setopt(pycurl.HEADERFUNCTION, hdr.write)

    print curl, url, header
    try:
        curl.perform()
    except pycurl.error, e:
        raise e    

    http_code = curl.getinfo(pycurl.HTTP_CODE)
    if http_code != 200:
        status_line = hdr.getvalue().splitlines()[0]
        status_message = status_line
        e =urllib2.HTTPError (str(url), http_code, status_message, {}, None)
        e.url = url
        raise e
    else:
        return content.getvalue()

self.set_option(pycurl.MAXREDIRS, 5)
        self.set_option(pycurl.NOSIGNAL, 1)
        # Setting this option with even a nonexistent file makes libcurl
        # handle cookie capture and playback automatically.
        self.set_option(pycurl.COOKIEFILE, "/dev/null")
        # Set timeouts to avoid hanging too long
        self.set_timeout(30)
        # Use password identification from .netrc automatically
        self.set_option(pycurl.NETRC, 1)
        # Set up a callback to capture the payload
        def payload_callback(x):
            self.payload += x
        self.set_option(pycurl.WRITEFUNCTION, payload_callback)
        def header_callback(x):
            self.hdr += x
        self.set_option(pycurl.HEADERFUNCTION, header_callback)

self.report.set_status(Report.PASSED)
            self.report.add('request_url', request_url)
            self.report.add('request_method', method)
            self.report.add('request_headers', json.dumps(dict(kwargs.get('headers', {}))))
            try:
                resp_buff_hdrs = BytesIO()
                resp_buff_body = BytesIO()
                buffer = BytesIO()
                _curl = init_pycurl()
                _curl.setopt(pycurl.URL, self.format_pycurl_url(request_url))
                _curl.setopt(pycurl.HEADERFUNCTION, self.header_function)
                _curl.setopt(pycurl.HTTPHEADER, self.format_pycurl_header(kwargs.get('headers', {})))
                _curl.setopt(pycurl.POST, len(kwargs.get('data', {}).items()))
                _curl.setopt(pycurl.CUSTOMREQUEST, method)
                _curl.setopt(pycurl.POSTFIELDS, urllib.parse.urlencode(kwargs.get('data', {})))
                _curl.setopt(pycurl.HEADERFUNCTION, resp_buff_hdrs.write)
                _curl.setopt(pycurl.WRITEFUNCTION, resp_buff_body.write)
                for retries in reversed(range(0, 3)):
                    try:
                        _curl.perform()
                    except Exception as e:
                        if retries:
                            self.logger.error('Retrying... ({}) because {}'.format(retries, e))
                        else:
                            raise e
                _return = Return()
                _return.status_code = _curl.getinfo(pycurl.RESPONSE_CODE)
                _return.headers = self.resp_headers
                _return.content = buffer.getvalue()
                _return.request = Return()
                _return.request.headers = kwargs.get('headers', {})
                _return.request.body = kwargs.get('data', {})