Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def run(self):
while 1:
try:
url, filename = self.queue.get_nowait()
except Queue.Empty:
raise SystemExit
fp = open(filename, "wb")
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.WRITEDATA, fp)
try:
curl.perform()
except:
import traceback
traceback.print_exc(file=sys.stderr)
sys.stderr.flush()
curl.close()
fp.close()
sys.stdout.write(".")
sys.stdout.flush()
curl = pycurl.Curl()
try:
if querydata:
if 'path' in querydata:
querydata['path'] = querydata['path'].encode('UTF-8')
url += '?%s' % urlencode(querydata)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.CONNECTTIMEOUT, 15)
curl.setopt(pycurl.LOW_SPEED_LIMIT, 1)
curl.setopt(pycurl.LOW_SPEED_TIME, 30)
curl.setopt(pycurl.USERAGENT, '')
curl.setopt(pycurl.HEADER, 0)
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.WRITEFUNCTION, self.__write_data)
starthour, endhour = SyncY.config['speedlimitperiod'].split('-', 1)
starthour = int(starthour)
endhour = int(endhour)
curhour = time.localtime().tm_hour
if (endhour > starthour and starthour <= curhour < endhour) or (endhour < starthour and (curhour < starthour or curhour >= endhour)):
curl.setopt(pycurl.MAX_SEND_SPEED_LARGE, SyncY.config['maxsendspeed'] / SyncY.config['tasknumber'] / SyncY.config['threadnumber'])
curl.setopt(pycurl.MAX_RECV_SPEED_LARGE, SyncY.config['maxrecvspeed'] / SyncY.config['tasknumber'] / SyncY.config['threadnumber'])
if self.__op == SYCurl.Upload:
curl.setopt(pycurl.UPLOAD, 1)
curl.setopt(pycurl.READFUNCTION, self.__read_data)
curl.setopt(pycurl.INFILESIZE, self.__endpos - startpos + 1)
with open(fnname, 'rb') as self.__fd:
self.__fd.seek(startpos)
flock(self.__fd, LOCK_SH)
of body_output and headers_output
:param url: resource url
:return: an ``(opener, body_output, headers_output)`` tuple.
"""
# http://curl.haxx.se/mail/curlpython-2005-06/0004.html
# http://curl.haxx.se/mail/lib-2010-03/0114.html
opener = opener or pycurl.Curl()
if getattr(opener, "dirty", True):
opener = self.clean_opener(opener)
logger.debug("Open url: %s" % url)
opener.setopt(pycurl.URL, url)
opener.setopt(pycurl.NOSIGNAL, 1)
if isinstance(self._auth, AuthManager):
self._auth.setup_request(self)
self._auth.setup(opener)
elif self._netrc:
self.setup_netrc(opener)
else:
opener.unsetopt(pycurl.USERPWD)
if self._headers:
logger.debug("Setup custom headers %s" %
"\r\n".join(["%s: %s" % (f, v) for f, v
in CaseInsensitiveDict(self._headers).iteritems()]))
opener.setopt(pycurl.HTTPHEADER, ["%s: %s" % (capwords(f, "-"), v) for f, v
in CaseInsensitiveDict(self._headers).iteritems()])
'Connection: Keep-Alive',
'Content-type: application/x-www-form-urlencoded; charset=UTF-8',
'Cache-control: no-cache',
'Pragma: no-cache',
'Pragma-directive: no-cache',
'Cache-directive: no-cache',
'Expires: 0']
c.setopt(pycurl.FOLLOWLOCATION, 1) # set follow redirects
c.setopt(pycurl.MAXREDIRS, 10) # set max redirects
c.setopt(pycurl.SSL_VERIFYHOST, 0) # don't verify host
c.setopt(pycurl.SSL_VERIFYPEER, 0) # don't verify peer
# c.setopt(pycurl.SSLVERSION, pycurl.SSLVERSION_SSLv3) # sslv3
c.setopt(pycurl.COOKIEFILE, '/dev/null') # black magic
c.setopt(pycurl.COOKIEJAR, '/dev/null') # black magic
c.setopt(pycurl.FRESH_CONNECT, 1) # important: no cache!
c.setopt(pycurl.NOSIGNAL, 1) # pass 'long' to stack to fix libcurl bug
c.setopt(pycurl.ENCODING, "") # use all available encodings (black magic)
if options.xforw: # set x-forwarded-for
generate_random_xforw = RandomIP()
xforwip = generate_random_xforw._generateip('')
xforwfakevalue = ['X-Forwarded-For: ' + str(xforwip)]
fakeheaders = fakeheaders + xforwfakevalue
if options.xclient: # set x-client-ip
generate_random_xclient = RandomIP()
xclientip = generate_random_xclient._generateip('')
xclientfakevalue = ['X-Client-IP: ' + str(xclientip)]
fakeheaders = fakeheaders + xclientfakevalue
if options.host: # set http host header
host_fakevalue = ['Host: ' + str(options.host)]
fakeheaders = fakeheaders + host_fakevalue
c.setopt(pycurl.HTTPHEADER, fakeheaders) # set fake headers
b = io.BytesIO()
def init_curl(self):
self.rep = StringIO()
self.header = ""
self.pycurl = pycurl.Curl()
self.pycurl.setopt(pycurl.FOLLOWLOCATION, 1)
self.pycurl.setopt(pycurl.MAXREDIRS, 5)
self.pycurl.setopt(pycurl.TIMEOUT, (self.timeout*3600))
self.pycurl.setopt(pycurl.CONNECTTIMEOUT, 30)
self.pycurl.setopt(pycurl.NOSIGNAL, 1)
self.pycurl.setopt(pycurl.NOPROGRESS, 0)
self.pycurl.setopt(pycurl.PROGRESSFUNCTION, self.progress)
self.pycurl.setopt(pycurl.AUTOREFERER, 1)
self.pycurl.setopt(pycurl.BUFFERSIZE, self.bufferSize)
self.pycurl.setopt(pycurl.SSL_VERIFYPEER, 0)
if self.debug:
self.pycurl.setopt(pycurl.VERBOSE, 1)
if self.interface:
self.pycurl.setopt(pycurl.INTERFACE, self.interface)
resolve_port = 80
resolve = '%s:%s:%s' % (resolve_host, resolve_port, resolve_ip)
else:
resolve = ''
if proxy_type in PROXY_TYPE_MAP:
proxy_type = PROXY_TYPE_MAP[proxy_type]
else:
raise ValueError('Invalid proxy_type %r' % proxy_type)
fp = pycurl.Curl()
fp.setopt(pycurl.SSL_VERIFYPEER, 0)
fp.setopt(pycurl.SSL_VERIFYHOST, 0)
fp.setopt(pycurl.HEADER, 1)
fp.setopt(pycurl.USERAGENT, 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0')
fp.setopt(pycurl.NOSIGNAL, 1)
fp.setopt(pycurl.FOLLOWLOCATION, int(follow))
fp.setopt(pycurl.MAXREDIRS, int(max_follow))
fp.setopt(pycurl.CONNECTTIMEOUT, int(timeout_tcp))
fp.setopt(pycurl.TIMEOUT, int(timeout))
fp.setopt(pycurl.PROXY, proxy)
fp.setopt(pycurl.PROXYTYPE, proxy_type)
fp.setopt(pycurl.RESOLVE, [resolve])
headers_output, body_output = StringIO(), StringIO()
fp.setopt(pycurl.HEADERFUNCTION, headers_output.write)
fp.setopt(pycurl.HEADER, 0)
fp.setopt(pycurl.WRITEFUNCTION, body_output.write)
def debug_func(t, s):
if max_mem > 0 and request.tell() > max_mem:
return 0
def set_opts(self, curl, url, params, headers,
ckey=None, cert=None, post=None, doseq=True, verbose=None):
"""Set options for given curl object"""
curl.setopt(pycurl.NOSIGNAL, self.nosignal)
curl.setopt(pycurl.TIMEOUT, self.timeout)
curl.setopt(pycurl.CONNECTTIMEOUT, self.connecttimeout)
curl.setopt(pycurl.FOLLOWLOCATION, self.followlocation)
curl.setopt(pycurl.MAXREDIRS, self.maxredirs)
curl.setopt(pycurl.COOKIEJAR, '.cookie')
curl.setopt(pycurl.COOKIEFILE, '.cookie')
encoded_data = urllib.urlencode(params, doseq=doseq)
if not post:
url = url + '?' + encoded_data
curl.setopt(pycurl.POST, 0)
if post:
curl.setopt(pycurl.POST, 1)
curl.setopt(pycurl.POSTFIELDS, encoded_data)
if isinstance(url, str):
curl.setopt(pycurl.URL, url)
def set_opts(self, curl, url, params, headers,
ckey=None, cert=None, capath=None, verbose=None,
verb='GET', doseq=True, encode=False, cainfo=None, cookie=None):
"""Set options for given curl object, params should be a dictionary"""
if not (isinstance(params, (dict, basestring)) or params is None):
raise TypeError("pycurl parameters should be passed as dictionary or an (encoded) string")
curl.setopt(pycurl.NOSIGNAL, self.nosignal)
curl.setopt(pycurl.TIMEOUT, self.timeout)
curl.setopt(pycurl.CONNECTTIMEOUT, self.connecttimeout)
curl.setopt(pycurl.FOLLOWLOCATION, self.followlocation)
curl.setopt(pycurl.MAXREDIRS, self.maxredirs)
# also accepts encoding/compression algorithms
if headers and headers.get("Accept-Encoding"):
if isinstance(headers["Accept-Encoding"], basestring):
curl.setopt(pycurl.ENCODING, headers["Accept-Encoding"])
else:
logging.warning("Wrong data type for header 'Accept-Encoding': %s",
type(headers["Accept-Encoding"]))
if cookie and url in cookie:
curl.setopt(pycurl.COOKIEFILE, cookie[url])
curl.setopt(pycurl.COOKIEJAR, cookie[url])
def _prepare_curl_callback(curl, next_callback):
curl.setopt(pycurl.NOSIGNAL, 1)
if callable(next_callback):
next_callback(curl)
def get_curl(url):
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
curl.setopt(pycurl.NOSIGNAL, 1)
return curl