Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
sys.stderr.write("Zookeeper must be specified\n")
usage()
sys.exit(3)
if not znode:
sys.stderr.write("Znode must be specified\n")
usage()
sys.exit(4)
if (not watch and not create and not exists):
sys.stderr.write("Exactly one of watch (-w), create (-c) or exists (-e) must be specified\n")
usage()
sys.exit(5)
while True:
zk = KazooClient(hosts=zookeeper, timeout=1, connection_retry=KazooRetry(max_tries=-1))
try:
zk.start()
if create:
try:
zk.create(znode)
except NodeExistsError:
pass
sys.exit(0)
elif watch:
while not zk.exists(znode):
print("Waiting for %s" % znode)
time.sleep(1)
sys.exit(0)
elif exists:
if zk.exists(znode):
sys.exit(0)
old_retry_keys[key] = kwargs.pop(key)
warnings.warn(
'Passing retry configuration param %s to the '
'client directly is deprecated, please pass a '
'configured retry object (using param %s)' % (
key, _RETRY_COMPAT_MAPPING[key]),
DeprecationWarning, stacklevel=2)
except KeyError:
pass
retry_keys = {}
for oldname, value in old_retry_keys.items():
retry_keys[_RETRY_COMPAT_MAPPING[oldname]] = value
if self._conn_retry is None:
self._conn_retry = KazooRetry(
sleep_func=self.handler.sleep_func,
**retry_keys)
if self.retry is None:
self.retry = KazooRetry(
sleep_func=self.handler.sleep_func,
**retry_keys)
# Managing legacy SASL options
for scheme, auth in self.auth_data:
if scheme != 'sasl':
continue
if sasl_options:
raise ConfigurationError(
'Multiple SASL configurations provided'
)
warnings.warn(
def create_zk_client(zk_hosts: str) -> KazooClient:
conn_retry_policy = KazooRetry(max_tries=-1, delay=0.1, max_delay=0.1)
cmd_retry_policy = KazooRetry(
max_tries=3, delay=0.3, backoff=1, max_delay=1, ignore_expire=False)
return KazooClient(
hosts=zk_hosts,
connection_retry=conn_retry_policy,
command_retry=cmd_retry_policy,
)
def __init__(self, client, path, func=None):
"""Create a data watcher for an existing path"""
self._client = client
self._path = path
self._func = func
self._stopped = False
self._run_lock = client.handler.lock_object()
self._version = None
self._retry = kazoo.retry.KazooRetry(
max_tries=None, sleep_func=client.handler.sleep_func
)
self._used = False
# Register our session listener if we're going to resume
# across session losses
if func is not None:
self._used = True
self._client.add_listener(self._session_watcher)
self._get_data()
def main():
""" Starts the groomer. """
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true',
help='Output debug-level logging')
args = parser.parse_args()
if args.verbose:
logger.setLevel(logging.DEBUG)
zk_hosts = appscale_info.get_zk_node_ips()
zk_client = KazooClient(hosts=','.join(zk_hosts),
connection_retry=ZK_PERSISTENT_RECONNECTS,
command_retry=KazooRetry(max_tries=-1))
zk_client.start()
db_access = DatastoreProxy()
thread_pool = ThreadPoolExecutor(4)
TransactionGroomer(zk_client, db_access, thread_pool)
logger.info('Starting transaction groomer')
IOLoop.current().start()
up to 3 seconds. If a command fails, that command is retried every 300ms for 3 attempts before failing.
These values are chosen to suit a human-interactive time.
Args:
zk_user:
The username to use when connecting to ZooKeeper or `None` if no authentication is necessary.
zk_secret:
The secret to use when connecting to ZooKeeper or `None` if no authentication is necessary.
Returns:
A ZooKeeper client connection in the form of a `kazoo.client.KazooClient`.
"""
# Try to reconnect indefinitely, with time between updates going
# exponentially to ~3s. Then every retry occurs every ~3 seconds.
conn_retry_policy = KazooRetry(
max_tries=-1,
delay=0.3,
backoff=1.3,
max_jitter=1,
max_delay=3,
ignore_expire=True,
)
# Retry commands every 0.3 seconds, for a total of <1s (usually 0.9)
cmd_retry_policy = KazooRetry(
max_tries=3,
delay=0.3,
backoff=1,
max_jitter=0.1,
max_delay=1,
ignore_expire=False,
)
def start(self):
"""Create a new group and wait until the partitions have been
acquired. This function should never be called twice.
:raises: PartitionerError upon partitioner failures
.. note: This is a blocking operation.
"""
self.kazoo_retry = KazooRetry(**KAZOO_RETRY_DEFAULTS)
self.kazoo_client = KazooClient(
self.config.zookeeper,
connection_retry=self.kazoo_retry,
)
self.kafka_client = KafkaClient(self.config.broker_list)
self.log.debug("Starting a new group for topics %s", self.topics)
self.released_flag = True
self._refresh()
GO = "go"
# PHP programs.
PHP = "php"
# Location where applications are stored.
APPS_PATH = "/var/apps/"
# Locations of ZooKeeper.
ZK_LOCATIONS_FILE = "/etc/appscale/zookeeper_locations"
# Default location for connecting to ZooKeeper.
ZK_DEFAULT_CONNECTION_STR = "localhost:2181"
# A ZooKeeper reconnect policy that never stops retrying to connect.
ZK_PERSISTENT_RECONNECTS = KazooRetry(max_tries=-1, max_delay=30)
# Default location for the datastore master.
MASTERS_FILE_LOC = "/etc/appscale/masters"
# Default location for the datastore slaves.
SLAVES_FILE_LOC = "/etc/appscale/slaves"
# Application ID for AppScale Dashboard.
DASHBOARD_APP_ID = "appscaledashboard"
# Reserved application identifiers which are only internal for AppScale.
RESERVED_APP_IDS = [DASHBOARD_APP_ID]
# The seconds to wait for the schema to settle after changing it.
SCHEMA_CHANGE_TIMEOUT = 120
self.wake_event = client.handler.event_object()
# props to Netflix Curator for this trick. It is possible for our
# create request to succeed on the server, but for a failure to
# prevent us from getting back the full path name. We prefix our
# lock name with a uuid and can check for its presence on retry.
self.prefix = uuid.uuid4().hex + self._NODE_NAME
self.create_path = self.path + "/" + self.prefix
self.create_tried = False
self.is_acquired = False
self.assured_path = False
self.cancelled = False
self.node = None
self._retry = KazooRetry(max_tries=None)
def __init__(self):
retry_policy = KazooRetry(max_tries=self.max_tries)
self.zk_client = self.get_kazoo_client(command_retry=retry_policy)
self.zk_client.start()
self.register_signal_handlers()