Juan Hernandez has uploaded a new change for review.
Change subject: [WIP] Replace httplib with pycurl
......................................................................
[WIP] Replace httplib with pycurl
Currently we use the Python httplib module to handle the HTTP connection
with the server. Unfortunatelly this module doesn't support any
interesting authentication mechanism. In the future we want to add
support for Kerberos, for example. As a first step this patch replaces
httplib with the pycurl.
Change-Id: I7fa93cef1b8ca0deb5bdbc02c35ab604e11b4570
Signed-off-by: Juan Hernandez <juan.hernan...@redhat.com>
---
M ovirt-engine-sdk-python.spec.in
M setup.py
M src/ovirtsdk/infrastructure/connectionspool.py
M src/ovirtsdk/infrastructure/proxy.py
M src/ovirtsdk/web/connection.py
D src/ovirtsdk/web/cookiejaradapter.py
6 files changed, 117 insertions(+), 296 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/ovirt-engine-sdk refs/changes/64/33064/1
diff --git a/ovirt-engine-sdk-python.spec.in b/ovirt-engine-sdk-python.spec.in
index bf5db54..5687e3b 100644
--- a/ovirt-engine-sdk-python.spec.in
+++ b/ovirt-engine-sdk-python.spec.in
@@ -13,6 +13,7 @@
Requires: python
Requires: python-lxml
+Requires: python-pycurl
Provides: ovirt-engine-sdk = %{version}-%{release}
Obsoletes: ovirt-engine-sdk < 3.3.0.6
diff --git a/setup.py b/setup.py
index 1caa8e5..2281d70 100755
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@
package_dir={ '': 'src' },
packages=[ 'ovirtsdk.infrastructure', 'ovirtsdk.utils', 'ovirtsdk.web',
'ovirtsdk.xml'],
py_modules=['ovirtsdk.api'],
- install_requires=['lxml >= 2.2.3'],
+ install_requires=['lxml >= 2.2.3', 'pycurl'],
entry_points={},
**version_info
)
diff --git a/src/ovirtsdk/infrastructure/connectionspool.py
b/src/ovirtsdk/infrastructure/connectionspool.py
index cc179cf..07027df 100644
--- a/src/ovirtsdk/infrastructure/connectionspool.py
+++ b/src/ovirtsdk/infrastructure/connectionspool.py
@@ -14,16 +14,13 @@
# limitations under the License.
#
-from Queue import Queue
+import pycurl
import thread
-import cookielib
-import urlparse
-from cookielib import DefaultCookiePolicy
+from Queue import Queue
from ovirtsdk.web.connection import Connection
from ovirtsdk.infrastructure.errors import ImmutableError
-from ovirtsdk.utils.synchronizationhelper import synchronized
class ConnectionsPool(object):
@@ -43,16 +40,14 @@
self.__url = url
self.__context = context
- # Create the cookies policy and jar:
- self.__cookies_jar = cookielib.CookieJar(
- policy=cookielib.DefaultCookiePolicy(
- strict_ns_domain=DefaultCookiePolicy.DomainStrictNoDots,
- allowed_domains=self.__getAllowedDomains(url)
- )
- )
+ # Create the objects shared by all the curl handles:
+ share = pycurl.CurlShare()
+ share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
+ share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
for _ in range(count):
- self.__free_connections.put(item=Connection(url=url, \
+ self.__free_connections.put(item=Connection(share=share,
+ url=url, \
port=port, \
key_file=key_file, \
cert_file=cert_file, \
@@ -65,6 +60,7 @@
insecure=insecure,
validate_cert_chain=validate_cert_chain,
debug=debug))
+
def getConnection(self, get_ttl=100):
# try:
with self.__plock:
@@ -79,47 +75,12 @@
# TODO: add more connections if needed
# continue
- def getCookiesJar(self):
- """
- returns cookies container
- """
- return self.__cookies_jar
-
- @synchronized
- def addCookieHeaders(self, request_adapter):
- """
- Adds the stored cookie/s to the request adapter:
- """
- self.getCookiesJar().add_cookie_header(request_adapter)
-
- @synchronized
- def storeCookies(self, response_adapter, request_adapter):
- """
- Stores the cookie/s located in the response adapter in request adapter:
- """
- self.getCookiesJar().extract_cookies(response_adapter, request_adapter)
-
def get_url(self):
return self.__url
@property
def context(self):
return self.__context
-
- @synchronized
- def __getAllowedDomains(self, url):
- '''
- fetches allowed domains for cookie
- '''
-
- LOCAL_HOST = 'localhost'
- parsed_url = urlparse.urlparse(url)
- domains = [parsed_url.hostname]
-
- if parsed_url.hostname == LOCAL_HOST:
- return domains.append(LOCAL_HOST + '.local')
-
- return domains
def _freeResource(self, conn):
with self.__rlock:
diff --git a/src/ovirtsdk/infrastructure/proxy.py
b/src/ovirtsdk/infrastructure/proxy.py
index a11b754..5016482 100644
--- a/src/ovirtsdk/infrastructure/proxy.py
+++ b/src/ovirtsdk/infrastructure/proxy.py
@@ -14,7 +14,7 @@
# limitations under the License.
#
-from ovirtsdk.infrastructure.errors import RequestError, ConnectionError,
FormatError
+from ovirtsdk.infrastructure.errors import FormatError
from ovirtsdk.xml import params
from lxml import etree
@@ -32,10 +32,6 @@
self.__connections_pool = connections_pool
self._persistent_auth = persistent_auth
self.__prefix = prefix
-
- # In order to create the cookies adapter we need to extract from the
- # URL the host name, so that we can accept cookies only from that host:
- self._url = self.__connections_pool.get_url()
def getConnectionsPool(self):
'''
@@ -139,7 +135,7 @@
response = conn.doRequest(
method=method,
- url=self.__prefix + url,
+ url=url,
body=body,
headers=headers,
last=last,
diff --git a/src/ovirtsdk/web/connection.py b/src/ovirtsdk/web/connection.py
index 9b0f4d6..445e0f0 100644
--- a/src/ovirtsdk/web/connection.py
+++ b/src/ovirtsdk/web/connection.py
@@ -14,18 +14,12 @@
# limitations under the License.
#
-import types
-import urllib
-import urlparse
-import base64
-import socket
+import cStringIO
import logging
+import pycurl
+import types
-from httplib import HTTPConnection
-
-from ovirtsdk.web.cookiejaradapter import CookieJarAdapter
from ovirtsdk.infrastructure.context import context
-from ovirtsdk.web.httpsconnection import HTTPSConnection
from ovirtsdk.infrastructure.errors import NoCertificatesError,
ImmutableError, RequestError, ConnectionError
@@ -33,54 +27,37 @@
'''
The oVirt api connection proxy
'''
- def __init__(self, url, port, key_file, cert_file, ca_file, strict,
timeout, username,
+ def __init__(self, share, url, port, key_file, cert_file, ca_file, strict,
timeout, username,
password, manager, insecure=False, validate_cert_chain=True,
debug=False):
- self.__connection = self.__createConnection(url=url,
- port=port,
- key_file=key_file,
- cert_file=cert_file,
- ca_file=ca_file,
- insecure=insecure,
-
validate_cert_chain=validate_cert_chain,
- strict=strict,
- timeout=timeout)
+ self.__curl = self.__create_curl(share=share,
+ url=url,
+ port=port,
+ key_file=key_file,
+ cert_file=cert_file,
+ ca_file=ca_file,
+ insecure=insecure,
+
validate_cert_chain=validate_cert_chain,
+ strict=strict,
+ timeout=timeout,
+ debug=debug)
self.__url = url
- self.__connection.set_debuglevel(int(debug))
- self.__headers = self.__createStaticHeaders(username, password)
self.__manager = manager
self.__id = id(self)
self.__insecure = insecure
self.__validate_cert_chain = validate_cert_chain
self.__context = manager.context
+ self.__username = username
+ self.__password = password
def get_id(self):
return self.__id
def getConnection(self):
- return self.__connection
+ return self.__curl
- def getDefaultHeaders(self, no_auth=False):
- '''
- Fetches headers to be used on request
-
- @param no_auth: do not authorize request (authorization is done via
cookie)
- '''
-
- AUTH_HEADER = 'Authorization'
-
- headers = self.__headers.copy()
- headers.update(self.__createDynamicHeaders())
- renew_session = context.manager[self.context].get('renew_session')
-
- # remove AUTH_HEADER
- if no_auth and not renew_session and headers.has_key(AUTH_HEADER):
- headers.pop(AUTH_HEADER)
-
- return headers
-
- def doRequest(self, method, url, body=urllib.urlencode({}), headers={},
last=False, persistent_auth=True):
+ def doRequest(self, method, url, body=None, headers={}, last=False,
persistent_auth=True):
'''
Performs HTTP request
@@ -93,161 +70,135 @@
'''
try:
- # Copy request headers to avoid by-ref lookup after
- # JSESSIONID has been injected
- request_headers = headers.copy()
+ # Set the method:
+ self.__curl.setopt(pycurl.CUSTOMREQUEST, method)
- # Add cookie headers as needed:
- request_adapter = CookieJarAdapter(self.__url + url,
request_headers)
- self.__manager.addCookieHeaders(request_adapter)
+ # Set the URL:
+ self.__curl.setopt(pycurl.URL, self.__url + url)
+
+ # Credentials should be sent only if there isn't a session:
+ if not self.__in_session():
+ self.__curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
+ self.__curl.setopt(pycurl.USERPWD, "%s:%s" % (self.__username,
self.__password))
+
+ # Default headers:
+ header_lines = []
+ for header in headers.items():
+ header_lines.append("%s: %s" % header)
+ header_lines.append("Content-Type: application/xml")
+ header_lines.append("Accept: application/xml")
# Every request except the last one should indicate that we prefer
# to use persistent authentication:
if persistent_auth and not last:
- request_headers["Prefer"] = "persistent-auth"
+ header_lines.append("Prefer: persistent-auth")
+
+ # Copy headers and the request body to the curl object:
+ self.__curl.setopt(pycurl.HTTPHEADER, header_lines)
+ if body is not None:
+ self.__curl.setopt(pycurl.POSTFIELDS, body)
+
+ # Prepare the buffer to receive the response body:
+ buffer = cStringIO.StringIO()
+ self.__curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
# Send the request and wait for the response:
- response = self.__connection.request(
- method,
- url,
- body,
- self.getHeaders(request_headers,
- no_auth=
- persistent_auth and \
- self.__isSetJsessionCookie(
-
self.__manager.getCookiesJar()
- ),
- )
- )
-
- response = self.getResponse()
-
- # Read the response headers (there is always a response,
- # even for error responses):
- response_headers = dict(response.getheaders())
+ self.__curl.perform()
# Parse the received body only if there are no errors reported by
- # the server (this needs review, as less than 400 doesn't garantee
+ # the server (this needs review, as less than 400 doesn't guarantee
# a correct response, it could be a redirect, and many other
# things):
- if response.status >= 400:
- raise RequestError, response
+ response_status = self.__curl.getinfo(pycurl.HTTP_CODE)
+ if response_status >= 400:
+ raise RequestError, response_status
- # Copy the cookies from the response:
- response_adapter = CookieJarAdapter(self.__url, response_headers)
- self.__manager.storeCookies(response_adapter, request_adapter)
+ return buffer.getvalue()
- # Parse the body:
- response_body = response.read()
-
- # Print response body (if in debug mode)
- self.__do_debug(self, response_body)
-
- return response_body
-
- except socket.error, e:
+ except pycurl.error, e:
raise ConnectionError, str(e)
finally:
self.close()
- def __do_debug(self, conn, body):
- '''
- Prints request body (when in debug) to STDIO
- '''
- if conn.getConnection().debuglevel:
- print 'body:\n' + body if body else ''
+ def __in_session(self):
+ """Checks if there is a session established."""
- def __isSetJsessionCookie(self, cookies_jar):
- '''
- Checks if JSESSIONID cookie is set
-
- @param cookies_jar: cookies container
- '''
- if cookies_jar and len(cookies_jar._cookies) > 0:
- for key in cookies_jar._cookies.keys():
- if key and len(cookies_jar._cookies[key]) > 0:
- for value in cookies_jar._cookies[key].values():
- if value and 'JSESSIONID' in value.keys():
- return True
+ for cookie_line in self.__curl.getinfo(pycurl.INFO_COOKIELIST):
+ cookie_fields = cookie_line.split("\t")
+ cookie_name = cookie_fields[5]
+ if cookie_name == "JSESSIONID":
+ return True
return False
- def getHeaders(self, headers={}, no_auth=False):
- headers.update(self.getDefaultHeaders(no_auth))
- extended_headers = {}
- for k in headers.keys():
- if (headers[k] is None and extended_headers.has_key(k)):
- extended_headers.pop(k)
- elif headers[k] != None:
- if type(headers[k]) != types.StringType:
- extended_headers[k] = str(headers[k])
- else:
- extended_headers[k] = headers[k]
-
- return extended_headers
-
def getResponse(self):
- return self.__connection.getresponse()
+ return self.__curl.getresponse()
def setDebugLevel(self, level):
- self.__connection.set_debuglevel(level)
+ self.__curl.set_debuglevel(level)
def setTunnel(self, host, port=None, headers=None):
- self.__connection.set_tunnel(host, port, headers)
+ self.__curl.set_tunnel(host, port, headers)
def close(self):
- self.__connection.close()
+ self.__curl.close()
# FIXME: create connection watchdog to close it on idle-ttl expiration, rather
than after the call
if (self.__manager is not None):
self.__manager._freeResource(self)
def state(self):
- return self.__connection.__state
+ return self.__curl.__state
@property
def context(self):
return self.__context
- def __parse_url(self, url):
- if not url.startswith('http'):
- url = "https://"; + url
- return urlparse.urlparse(url)
+ def __create_curl(self, share, url, key_file=None, cert_file=None,
+ ca_file=None, insecure=False, validate_cert_chain=True,
port=None,
+ strict=None, timeout=None, debug=False):
+ curl = pycurl.Curl()
+ curl.setopt(pycurl.SHARE, share)
+ curl.setopt(pycurl.COOKIEFILE, "/dev/null")
+ curl.setopt(pycurl.COOKIEJAR, "/dev/null")
-
- def __createConnection(self, url, key_file=None, cert_file=None,
- ca_file=None, insecure=False,
validate_cert_chain=True, port=None,
- strict=None, timeout=None):
-
- u = self.__parse_url(url)
-
- if(u.scheme == 'https'):
+ if url.startswith("https"):
if validate_cert_chain:
if not insecure and not ca_file:
raise NoCertificatesError
else:
ca_file = None
- return HTTPSConnection(
- host=u.hostname,
- port=u.port,
- key_file=key_file,
- cert_file=cert_file,
- ca_file=ca_file,
- strict=strict,
- timeout=timeout,
- insecure=insecure
- )
- return HTTPConnection(
- host=u.hostname,
- port=u.port,
- strict=strict,
- timeout=timeout
- )
+ curl.setopt(pycurl.SSL_VERIFYPEER, 0 if insecure else 1)
+ curl.setopt(pycurl.SSL_VERIFYHOST, 0 if insecure else 2)
+ if ca_file is not None:
+ curl.setopt(pycurl.CAINFO, ca_file)
+ if cert_file is not None and key_file is not None:
+ curl.setopt(pycurl.SSLCERTTYPE, "PEM")
+ curl.setopt(pycurl.SSLCERT, cert_file)
+ curl.setopt(pycurl.SSLKEY, key_file)
- def __createStaticHeaders(self, username, password):
- auth = base64.encodestring("%s:%s" % (username,
password)).replace("\n", "")
- return {"Content-type" : "application/xml",
- "Accept" : "application/xml",
- "Authorization": "Basic %s" % auth}
+ if timeout is not None:
+ curl.setopt(pycurl.TIMEOUT, timeout)
+
+ if debug:
+ curl.setopt(pycurl.VERBOSE, 1)
+ curl.setopt(pycurl.DEBUGFUNCTION, self.__curl_debug)
+
+ return curl
+
+ def __curl_debug(self, debug_type, debug_message):
+ prefix = "* "
+ if debug_type == pycurl.INFOTYPE_DATA_IN:
+ prefix = "< "
+ elif debug_type == pycurl.INFOTYPE_DATA_OUT:
+ prefix = "> "
+ elif debug_type == pycurl.INFOTYPE_HEADER_IN:
+ prefix = "< "
+ elif debug_type == pycurl.INFOTYPE_HEADER_OUT:
+ prefix = "> "
+ lines = debug_message.replace("\r\n", "\n").strip().split("\n")
+ for line in lines:
+ print("%s%s" % (prefix, line))
+
def __injectFilterHeader(self, headers):
filter_header = context.manager[self.context].get('filter')
@@ -275,14 +226,6 @@
)
)
- def __createDynamicHeaders(self):
- headers = {}
-
- self.__injectFilterHeader(headers)
- self.__injectAuthSessionHeader(headers)
-
- return headers
-
def __setattr__(self, name, value):
if name in ['__context', 'context']:
raise ImmutableError(name)
@@ -302,3 +245,4 @@
validate the server's certificate (default is True)
'''
return self.__validate_cert_chain
+
diff --git a/src/ovirtsdk/web/cookiejaradapter.py
b/src/ovirtsdk/web/cookiejaradapter.py
deleted file mode 100644
index 0d2f203..0000000
--- a/src/ovirtsdk/web/cookiejaradapter.py
+++ /dev/null
@@ -1,81 +0,0 @@
-
-#
-# Copyright (c) 2010 Red Hat, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import urlparse
-
-
-class CookieJarAdapter():
- """
- This class is an adapter that implements the methods that the CookieJar
- expects and needs in order to add the cookie headers to an HTTP request.
-
- From the point of view of the CookieJar it looks like a urllib2 request and
- also like a response, but it just saves the headers to a list that will
later
- be retrieved by the proxy.
- """
- def __init__(self, url, headers):
- # Save the URL and the headers:
- self._url = url
- self._headers = headers
-
- # Extract the scheme and the host name from the URL:
- parsed_url = urlparse.urlparse(self._url)
- self._scheme = parsed_url.scheme
- self._host = parsed_url.hostname
-
- # The following methods are needed to simulate the behaviour of an urllib2
- # request class:
- def get_full_url(self):
- return self._url
-
- def get_host(self):
- return self._host
-
- def get_type(self):
- return self._scheme
-
- def is_unverifiable(self):
- return False
-
- def get_origin_req_host(self):
- return self._host
-
- def has_header(self, header):
- return header.lower() in self._headers
-
- def get_header(self, header_name, default=None):
- return self._headers.get(header_name.lower(), default)
-
- def header_items(self):
- return self._headers.items()
-
- def add_unredirected_header(self, key, val):
- self._headers[key.lower()] = val
-
- # The following method is needed to simulate the behaviour of an urllib2
- # response class:
- def info(self):
- return self
-
- # This methods simulates the object returned by the info method of the
- # urllib2 response class:
- def getheaders(self, name):
- result = []
- for key, value in self._headers.items():
- if key.lower() == name.lower():
- result.append(value)
- return result
--
To view, visit http://gerrit.ovirt.org/33064
To unsubscribe, visit http://gerrit.ovirt.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7fa93cef1b8ca0deb5bdbc02c35ab604e11b4570
Gerrit-PatchSet: 1
Gerrit-Project: ovirt-engine-sdk
Gerrit-Branch: master
Gerrit-Owner: Juan Hernandez <juan.hernan...@redhat.com>
_______________________________________________
Engine-patches mailing list
Engine-patches@ovirt.org
http://lists.ovirt.org/mailman/listinfo/engine-patches
