How to use the mrjob.conf.ClearedValue function in mrjob

To help you get started, we’ve selected a few mrjob examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github Yelp / mrjob / tests / test_option_store.py View on Github external
def test_clear_setup(self):
        opts = self.opts_for_conf('extend.conf', {
            'include': self.base_conf_path,
            'runners': {
                'inline': {
                    'setup': ClearedValue(['instead do this'])
                }
            }
        })

        self.assertEqual(opts['cmdenv'], self.base_opts['cmdenv'])
        self.assertEqual(opts['jobconf'], self.base_opts['jobconf'])
        self.assertEqual(opts['setup'], ['instead do this'])
github Yelp / mrjob / tests / test_conf.py View on Github external
{'foo': ClearedValue(['bar', {'baz': 'qux'}])})

        self.assertEqual(
            _fix_clear_tags(
                {'foo': [ClearedValue('bar'), {'baz': 'qux'}]}),
            {'foo': ['bar', {'baz': 'qux'}]})

        self.assertEqual(
            _fix_clear_tags(
                {'foo': ['bar', ClearedValue({'baz': 'qux'})]}),
            {'foo': ['bar', {'baz': 'qux'}]})

        self.assertEqual(
            _fix_clear_tags(
                {'foo': ['bar', {ClearedValue('baz'): 'qux'}]}),
            {'foo': ['bar', {'baz': ClearedValue('qux')}]})

        self.assertEqual(
            _fix_clear_tags(
                {'foo': ['bar', {'baz': ClearedValue('qux')}]}),
            {'foo': ['bar', {'baz': ClearedValue('qux')}]})
github Yelp / mrjob / tests / test_conf.py View on Github external
def test_dict(self):
        self.assertEqual(_fix_clear_tags({'foo': 'bar'}), {'foo': 'bar'})

        self.assertEqual(_fix_clear_tags(ClearedValue({'foo': 'bar'})),
                         ClearedValue({'foo': 'bar'}))

        self.assertEqual(_fix_clear_tags({ClearedValue('foo'): 'bar'}),
                         {'foo': ClearedValue('bar')})

        self.assertEqual(_fix_clear_tags({'foo': ClearedValue('bar')}),
                         {'foo': ClearedValue('bar')})

        self.assertEqual(
            _fix_clear_tags(
                ClearedValue({ClearedValue('foo'): ClearedValue('bar')})),
            ClearedValue({'foo': ClearedValue('bar')}))

        # ClearedValue('foo') key overrides 'foo' key
        self.assertEqual(
            _fix_clear_tags({ClearedValue('foo'): 'bar', 'foo': 'baz'}),
            {'foo': ClearedValue('bar')})
github Yelp / mrjob / tests / test_conf.py View on Github external
def test_cleared_opt_values(self):
        self.assertEqual(
            combine_opts(dict(foo=combine_lists),
                         {'foo': ['bar']},
                         {'foo': ClearedValue(['baz'])}),
            # ClearedValue(['baz']) overrides bar
            {'foo': ['baz']})

        self.assertEqual(
            combine_opts(dict(foo=combine_lists),
                         {'foo': ['bar']},
                         {'foo': ClearedValue(None)}),
            # not None!
            {'foo': []})
github Yelp / mrjob / tests / test_conf.py View on Github external
def test_empty(self):
        self.assertEqual(_load_yaml_with_clear_tag(''),
                         None)
        self.assertEqual(_load_yaml_with_clear_tag('!clear'),
                         ClearedValue(None))
github Yelp / mrjob / tests / test_runner.py View on Github external
def test_clear_cmdenv(self):
        opts = self.opts_for_conf('extend.conf', {
            'include': self.base_conf_path,
            'runners': {
                'inline': {
                    'cmdenv': ClearedValue({
                        'USER': 'dave'
                    })
                }
            }
        })

        self.assertEqual(opts['cmdenv'], {'USER': 'dave'})
        self.assertEqual(opts['jobconf'], self.base_opts['jobconf'])
        self.assertEqual(opts['setup'], self.base_opts['setup'])
github Yelp / mrjob / tests / test_conf.py View on Github external
def test_clear_paths(self):
        self.assertEqual(
            combine_envs(
                {'PATH': '/bin:/usr/bin',
                 'PYTHONPATH': '/usr/lib/python/site-packages',
                 'PS1': '> '},
                {'PATH': ClearedValue('/home/dave/bin'),
                 'PYTHONPATH': ClearedValue(None),
                 'CLASSPATH': '/home/dave/java',
                 'PS1': '\w> '}),
            {'PATH': '/home/dave/bin',
             'CLASSPATH': '/home/dave/java',
             'PS1': '\w> '})
github Yelp / mrjob / tests / test_conf.py View on Github external
def test_cant_clear_entire_opt_dicts(self):
        self.assertRaises(
            TypeError,
            combine_opts,
            dict(foo=combine_lists),
            {'foo': ['bar']},
            ClearedValue({'foo': ['baz']}))
github Yelp / mrjob / mrjob / runner.py View on Github external
log.warning('Deprecated option %s (from %s) has been renamed'
                            ' to %s and will be removed in v0.7.0' % (
                                k, source, aliased_opt))

                if opts.get(aliased_opt) is not None:
                    return  # don't overwrite non-aliased opt

                k = aliased_opt

            if k in self.OPT_NAMES:
                if v is None:
                    fixed_v = None
                elif isinstance(v, ClearedValue):
                    # _fix_opt() doesn't need to know about !clear (see #2102)
                    fixed_v = ClearedValue(self._fix_opt(k, v.value, source))
                else:
                    fixed_v = self._fix_opt(k, v, source)

                results[k] = fixed_v
            elif v:
                log.warning('Unexpected option %s (from %s)' % (k, source))

        return results
github Yelp / mrjob / mrjob / conf.py View on Github external
if isinstance(x, list):
        return [_fix(_strip_clear_tag(item)) for item in x]

    elif isinstance(x, dict):
        d = dict((_fix(k), _fix(v)) for k, v in x.items())

        # handle cleared keys
        for k, v in list(d.items()):
            if isinstance(k, ClearedValue):
                del d[k]
                d[_strip_clear_tag(k)] = ClearedValue(_strip_clear_tag(v))

        return d

    elif isinstance(x, ClearedValue):
        return ClearedValue(_fix(x.value))

    else:
        return x