How to use the mrjob.compat.map_version function in mrjob

To help you get started, we’ve selected a few mrjob examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github Yelp / mrjob / tests / mock_boto3 / emr.py View on Github external
def _add_steps(self, operation_name, Steps, cluster, now=None):
        if now is None:
            now = _boto3_now()

        _validate_param_type(Steps, (list, tuple))

        # only active job flows allowed
        if cluster['Status']['State'].startswith('TERMINAT'):
            raise _ValidationException(
                operation_name,
                'A job flow that is shutting down, terminated, or finished'
                ' may not be modified.')

        # no more than 256 steps allowed
        if cluster.get('RunningAmiVersion') and map_version(
                cluster['RunningAmiVersion'],
                LIFETIME_STEP_LIMIT_AMI_VERSIONS):
            # for very old AMIs, *all* steps count
            if len(cluster['_Steps']) + len(Steps) > STEP_ADD_LIMIT:
                raise _ValidationException(
                    operation_name,
                    'Maximum number of steps for job flow exceeded')
        else:
            # otherwise, only active and pending steps count
            num_active_steps = sum(
                1 for step in cluster['_Steps']
                if step['Status']['State'] in (
                    'PENDING', 'PENDING_CANCELLED', 'RUNNING'))

            if num_active_steps + len(Steps) > STEP_ADD_LIMIT:
                raise _ValidationException(
github Yelp / mrjob / tests / test_compat.py View on Github external
def test_list_of_tuples(self):
        version_map = [
            (LooseVersion('1'), 'foo'),
            (LooseVersion('2'), 'bar'),
            (LooseVersion('3'), 'baz'),
        ]

        self.assertEqual(map_version('1.1', version_map), 'foo')
        self.assertEqual(map_version('2', version_map), 'bar')
        self.assertEqual(map_version('4.5', version_map), 'baz')
        self.assertEqual(map_version('11.11', version_map), 'baz')
        self.assertEqual(map_version('0.1', version_map), 'foo')
github Yelp / mrjob / tests / test_compat.py View on Github external
def test_dict(self):
        version_map = {
            '1': 'foo',
            '2': 'bar',
            '3': 'baz',
        }

        self.assertEqual(map_version('1.1', version_map), 'foo')
        # test exact match
        self.assertEqual(map_version('2', version_map), 'bar')
        # versions are just minimums
        self.assertEqual(map_version('4.5', version_map), 'baz')
        # compare versions, not strings
        self.assertEqual(map_version('11.11', version_map), 'baz')
        # fall back to lowest version
        self.assertEqual(map_version('0.1', version_map), 'foo')
github Yelp / mrjob / tests / test_compat.py View on Github external
def test_dict(self):
        version_map = {
            '1': 'foo',
            '2': 'bar',
            '3': 'baz',
        }

        self.assertEqual(map_version('1.1', version_map), 'foo')
        # test exact match
        self.assertEqual(map_version('2', version_map), 'bar')
        # versions are just minimums
        self.assertEqual(map_version('4.5', version_map), 'baz')
        # compare versions, not strings
        self.assertEqual(map_version('11.11', version_map), 'baz')
        # fall back to lowest version
        self.assertEqual(map_version('0.1', version_map), 'foo')
github Yelp / mrjob / tests / test_compat.py View on Github external
def test_empty(self):
        self.assertRaises(ValueError, map_version, '0.5.0', None)
        self.assertRaises(ValueError, map_version, '0.5.0', {})
        self.assertRaises(ValueError, map_version, '0.5.0', [])
github Yelp / mrjob / tests / test_compat.py View on Github external
def test_list_of_tuples(self):
        version_map = [
            (LooseVersion('1'), 'foo'),
            (LooseVersion('2'), 'bar'),
            (LooseVersion('3'), 'baz'),
        ]

        self.assertEqual(map_version('1.1', version_map), 'foo')
        self.assertEqual(map_version('2', version_map), 'bar')
        self.assertEqual(map_version('4.5', version_map), 'baz')
        self.assertEqual(map_version('11.11', version_map), 'baz')
        self.assertEqual(map_version('0.1', version_map), 'foo')
github Yelp / mrjob / tests / test_compat.py View on Github external
def test_list_of_tuples(self):
        version_map = [
            (LooseVersion('1'), 'foo'),
            (LooseVersion('2'), 'bar'),
            (LooseVersion('3'), 'baz'),
        ]

        self.assertEqual(map_version('1.1', version_map), 'foo')
        self.assertEqual(map_version('2', version_map), 'bar')
        self.assertEqual(map_version('4.5', version_map), 'baz')
        self.assertEqual(map_version('11.11', version_map), 'baz')
        self.assertEqual(map_version('0.1', version_map), 'foo')
github Yelp / mrjob / tests / test_compat.py View on Github external
def test_empty(self):
        self.assertRaises(ValueError, map_version, '0.5.0', None)
        self.assertRaises(ValueError, map_version, '0.5.0', {})
        self.assertRaises(ValueError, map_version, '0.5.0', [])
github Yelp / mrjob / mrjob / emr.py View on Github external
def _ssh_tunnel_config(self):
        """Look up AMI version, and return a dict with the following keys:

        name: "job tracker" or "resource manager"
        path: path to start page of job tracker/resource manager
        port: port job tracker/resource manager is running on.
        """
        return map_version(self.get_image_version(),
                           _IMAGE_VERSION_TO_SSH_TUNNEL_CONFIG)