How to use the rucio.core.rse_expression_parser.parse_expression function in rucio

To help you get started, we’ve selected a few rucio examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github rucio / rucio / lib / rucio / core / rule.py View on Github external
# Resolve the new_child_dids to its locks
                with record_timer_block('rule.evaluate_did_attach.resolve_did_to_locks_and_replicas'):
                    # Resolve the rules to possible target rses:
                    possible_rses = []
                    source_rses = []
                    if session.bind.dialect.name != 'sqlite':
                        session.begin_nested()
                    try:
                        for rule in rules:
                            if rule.source_replica_expression:
                                source_rses.extend(parse_expression(rule.source_replica_expression, session=session))

                            if rule.ignore_availability:
                                possible_rses.extend(parse_expression(rule.rse_expression, session=session))
                            else:
                                possible_rses.extend(parse_expression(rule.rse_expression, filter={'availability_write': True}, session=session))
                    except Exception, e:
                        logging.warning('Could not parse RSE expression for possible RSEs for rule %s' % (str(rule.id)))
                        session.rollback()
                        possible_rses = []
                        session.begin_nested()
                    if session.bind.dialect.name != 'sqlite':
                        session.commit()

                    source_rses = list(set([rse['id'] for rse in source_rses]))
                    possible_rses = list(set([rse['id'] for rse in possible_rses]))

                    datasetfiles, locks, replicas, source_replicas = __resolve_dids_to_locks_and_replicas(dids=new_child_dids,
                                                                                                          nowait=True,
                                                                                                          restrict_rses=possible_rses,
                                                                                                          source_rses=source_rses,
                                                                                                          session=session)
github rucio / rucio / lib / rucio / core / permission / atlas.py View on Github external
:param kwargs: List of arguments for the action.
    :returns: True if account is allowed to call the API call, otherwise False
    """
    if _is_root(issuer):
        return True
    if get_rule(kwargs['rule_id'])['account'] == issuer:
        return True

    # Check if user is a country admin
    admin_in_country = []
    for kv in list_account_attributes(account=issuer):
        if kv['key'].startswith('country-') and kv['value'] == 'admin':
            admin_in_country.append(kv['key'].partition('-')[2])

    rule = get_rule(rule_id=kwargs['rule_id'])
    rses = parse_expression(rule['rse_expression'])
    if admin_in_country:
        for rse in rses:
            if list_rse_attributes(rse_id=rse['id']).get('country') in admin_in_country:
                return True

    # DELETERS can approve the rule
    for rse in rses:
        rse_attr = list_rse_attributes(rse_id=rse['id'])
        if rse_attr.get('rule_deleters'):
            if issuer.external in rse_attr.get('rule_deleters').split(','):
                return True

    return False
github rucio / rucio / lib / rucio / core / rule.py View on Github external
if ignore_availability:
                rses = parse_expression(rse_expression, session=session)
            else:
                rses = parse_expression(rse_expression, filter={'availability_write': True}, session=session)

            if lifetime is None:  # Check if one of the rses is a staging area
                if [rse for rse in rses if rse.get('staging_area', False)]:
                    raise StagingAreaRuleRequiresLifetime()

            # Auto-lock rules for TAPE rses
            if not locked:
                if [rse for rse in rses if rse.get('rse_type', RSEType.DISK) == RSEType.TAPE]:
                    locked = True

            if source_replica_expression:
                source_rses = parse_expression(source_replica_expression, session=session)
            else:
                source_rses = []

        # 2. Create the rse selector
        with record_timer_block('rule.add_rule.create_rse_selector'):
            rseselector = RSESelector(account=account, rses=rses, weight=weight, copies=copies, ignore_account_limit=ask_approval, session=session)

        expires_at = datetime.utcnow() + timedelta(seconds=lifetime) if lifetime is not None else None

        if notify == 'Y':
            notify = RuleNotification.YES
        elif notify == 'C':
            notify = RuleNotification.CLOSE
        else:
            notify = RuleNotification.NO
github rucio / rucio / lib / rucio / daemons / reaper / reaper.py View on Github external
if rses:
        invalid = set(rses) - set([rse['rse'] for rse in all_rses])
        if invalid:
            msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '',
                                                    ', '.join([repr(rse) for rse in invalid]))
            raise RSENotFound(msg)
        rses = [rse for rse in all_rses if rse['rse'] in rses]
    else:
        rses = all_rses

    if exclude_rses:
        excluded_rses = parse_expression(exclude_rses)
        rses = [rse for rse in rses if rse not in excluded_rses]

    if include_rses:
        included_rses = parse_expression(include_rses)
        rses = [rse for rse in rses if rse in included_rses]

    if not rses:
        logging.error('Reaper: No RSEs found. Exiting.')
        return

    logging.info('Reaper: This instance will work on RSEs: ' + ', '.join([rse['rse'] for rse in rses]))

    threads = []
    nb_rses_per_worker = int(math.ceil(len(rses) / float(total_workers))) or 1
    rses = random.sample(rses, len(rses))
    for worker in range(total_workers):
        for child in range(threads_per_worker or 1):
            rses_list = rses[worker * nb_rses_per_worker: worker * nb_rses_per_worker + nb_rses_per_worker]
            if not rses_list:
                logging.warning('Reaper: Empty RSEs list for worker %(worker)s' % locals())
github rucio / rucio / lib / rucio / core / permission / generic.py View on Github external
"""
    Checks if an account can delete a global account limit.

    :param issuer: Account identifier which issues the command.
    :param kwargs: List of arguments for the action.
    :returns: True if account is allowed, otherwise False
    """
    if _is_root(issuer) or has_account_attribute(account=issuer, key='admin'):
        return True
    # Check if user is a country admin
    admin_in_country = set()
    for kv in list_account_attributes(account=issuer):
        if kv['key'].startswith('country-') and kv['value'] == 'admin':
            admin_in_country.add(kv['key'].partition('-')[2])
    if admin_in_country:
        resolved_rse_countries = {list_rse_attributes(rse_id=rse['rse_id']).get('country') for rse in parse_expression(kwargs['rse_exp'])}
        if resolved_rse_countries.issubset(admin_in_country):
            return True
    return False
github rucio / rucio / lib / rucio / daemons / transmogrifier / transmogrifier.py View on Github external
attemptnr = 0
                                    skip_rule_creation = False

                                    selected_rses = []
                                    chained_idx = rule_dict.get('chained_idx', None)
                                    if chained_idx:
                                        params = {}
                                        if rule_dict.get('associated_site_idx', None):
                                            params['associated_site_idx'] = rule_dict.get('associated_site_idx', None)
                                        logging.debug('%s Chained subscription identified. Will use %s', prepend_str, str(created_rules[chained_idx]))
                                        algorithm = rule_dict.get('algorithm', None)
                                        selected_rses = select_algorithm(algorithm, created_rules[chained_idx], params)
                                    else:
                                        # In the case of chained subscription, don't use rseselector but use the rses returned by the algorithm
                                        if split_rule:
                                            rses = parse_expression(rse_expression)
                                            list_of_rses = [rse['id'] for rse in rses]
                                            # Check that some rule doesn't already exist for this DID and subscription
                                            preferred_rse_ids = []
                                            for rule in list_rules(filters={'subscription_id': subscription_id, 'scope': did['scope'], 'name': did['name']}):
                                                already_existing_rses = [(rse['rse'], rse['id']) for rse in parse_expression(rule['rse_expression'])]
                                                for rse, rse_id in already_existing_rses:
                                                    if (rse_id in list_of_rses) and (rse_id not in preferred_rse_ids):
                                                        preferred_rse_ids.append(rse_id)
                                            if len(preferred_rse_ids) >= copies:
                                                skip_rule_creation = True
                                            rse_id_dict = {}
                                            for rse in rses:
                                                rse_id_dict[rse['id']] = rse['rse']
                                            try:
                                                rseselector = RSESelector(account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids))
                                                selected_rses = [rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse(0, preferred_rse_ids=preferred_rse_ids, copies=copies, blacklist=blacklisted_rse_id)]
github rucio / rucio / lib / rucio / common / replica_sorter.py View on Github external
def site_selector(replicas, site):
    """
    Return a list of replicas located on one site.
    :param replicas : A dict with RSEs as values and replicas as keys (URIs).
    :param site : The name of the site
    """
    result = []
    try:
        rses = parse_expression("site=%s" % site)
    except InvalidRSEExpression:
        return result
    except Exception:
        return result
    rses = [i['rse'] for i in rses]
    for replica in replicas:
        if replicas[replica] in rses:
            result.append(replica)
    return result
github rucio / rucio / lib / rucio / core / replica.py View on Github external
:param sign_urls: If set, will sign the PFNs if necessary.
    :param signature_lifetime: If supported, in seconds, restrict the lifetime of the signed PFN.
    :param resolve_archives: When set to true, find archives which contain the replicas.
    :param resolve_parents: When set to true, find all parent datasets which contain the replicas.
    :param session: The database session in use.
    """

    file_clause, dataset_clause, state_clause, files, constituents = _resolve_dids(dids=dids, unavailable=unavailable,
                                                                                   ignore_availability=ignore_availability,
                                                                                   all_states=all_states,
                                                                                   resolve_archives=resolve_archives,
                                                                                   session=session)

    rse_clause = []
    if rse_expression:
        for rse in parse_expression(expression=rse_expression, session=session):
            rse_clause.append(models.RSEFileAssociation.rse_id == rse['id'])
    for f in _list_replicas(dataset_clause, file_clause, state_clause, pfns,
                            schemes, files, rse_clause, rse_expression, client_location, domain,
                            sign_urls, signature_lifetime, constituents, resolve_parents, session):
        yield f
github rucio / rucio / lib / rucio / daemons / conveyor / common.py View on Github external
"""
    Get a list of rses for conveyor

    :param rses:          List of rses
    :param include_rses:  RSEs to include
    :param exclude_rses:  RSEs to exclude
    :return:              List of working rses
    """
    working_rses = []
    rses_list = list_rses()
    if rses:
        working_rses = [rse for rse in rses_list if rse['rse'] in rses]

    if include_rses:
        try:
            parsed_rses = parse_expression(include_rses, session=None)
        except InvalidRSEExpression as error:
            logging.error("Invalid RSE exception %s to include RSEs", include_rses)
        else:
            for rse in parsed_rses:
                if rse not in working_rses:
                    working_rses.append(rse)

    if not (rses or include_rses):
        working_rses = rses_list

    if exclude_rses:
        try:
            parsed_rses = parse_expression(exclude_rses, session=None)
        except InvalidRSEExpression as error:
            logging.error("Invalid RSE exception %s to exclude RSEs: %s", exclude_rses, error)
        else: