Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def isna(obj):
"""
Detect missing values for an array-like object.
Args:
obj: Object to check for null or missing values.
Returns:
bool or array-like of bool
"""
if isinstance(obj, BasePandasDataset):
return obj.isna()
else:
return pandas.isna(obj)
Args:
to_append: The object to append to this.
ignore_index: Ignore the index on appending.
verify_integrity: Verify the integrity of the index on completion.
Returns:
A new DataFrame containing the concatenated values.
"""
from .dataframe import DataFrame
bad_type_msg = (
'cannot concatenate object of type "{}"; only pd.Series, '
"pd.DataFrame, and pd.Panel (deprecated) objs are valid"
)
if isinstance(to_append, list):
if not all(isinstance(o, BasePandasDataset) for o in to_append):
raise TypeError(
bad_type_msg.format(
type(
next(
o
for o in to_append
if not isinstance(o, BasePandasDataset)
)
)
)
)
elif all(isinstance(o, Series) for o in to_append):
self.name = None
for i in range(len(to_append)):
to_append[i].name = None
to_append[i] = to_append[i]._query_compiler
"Can only append a Series if ignore_index=True"
" or if the Series has a name"
)
if other.name is not None:
# other must have the same index name as self, otherwise
# index name will be reset
name = other.name
# We must transpose here because a Series becomes a new row, and the
# structure of the query compiler is currently columnar
other = other._query_compiler.transpose()
other.index = pandas.Index([name], name=self.index.name)
else:
# See note above about transpose
other = other._query_compiler.transpose()
elif isinstance(other, list):
if not all(isinstance(o, BasePandasDataset) for o in other):
other = DataFrame(pandas.DataFrame(other))._query_compiler
else:
other = [obj._query_compiler for obj in other]
else:
other = other._query_compiler
# If ignore_index is False, by definition the Index will be correct.
# We also do this first to ensure that we don't waste compute/memory.
if verify_integrity and not ignore_index:
appended_index = (
self.index.append(other.index)
if not isinstance(other, list)
else self.index.append([o.index for o in other])
)
is_valid = next((False for idx in appended_index.duplicated() if idx), True)
if not is_valid:
def _get_rename_function(mapper):
if isinstance(mapper, (dict, BasePandasDataset)):
def f(x):
if x in mapper:
return mapper[x]
else:
return x
else:
f = mapper
return f
def notna(obj):
if isinstance(obj, BasePandasDataset):
return obj.notna()
else:
return pandas.notna(obj)
import numpy as np
import sys
from typing import Tuple, Union
import warnings
from modin.error_message import ErrorMessage
from .utils import from_pandas, from_non_pandas, to_pandas, _inherit_docstrings
from .iterator import PartitionIterator
from .series import Series
from .base import BasePandasDataset
@_inherit_docstrings(
pandas.DataFrame, excluded=[pandas.DataFrame, pandas.DataFrame.__init__]
)
class DataFrame(BasePandasDataset):
def __init__(
self,
data=None,
index=None,
columns=None,
dtype=None,
copy=False,
query_compiler=None,
):
"""Distributed DataFrame object backed by Pandas dataframes.
Args:
data (numpy ndarray (structured or homogeneous) or dict):
Dict can contain Series, arrays, constants, or list-like
objects.
index (pandas.Index, list, ObjectID): The row index for this
# Reduce to a scalar if axis is None.
if level is not None:
return self._handle_level_agg(
axis, level, "any", skipna=skipna, **kwargs
)
else:
result = self._reduce_dimension(
self._query_compiler.any(
axis=0,
bool_only=bool_only,
skipna=skipna,
level=level,
**kwargs
)
)
if isinstance(result, BasePandasDataset):
return result.any(
axis=axis, bool_only=bool_only, skipna=skipna, level=level, **kwargs
)
return result
from .base import BasePandasDataset
from .iterator import PartitionIterator
from .utils import _inherit_docstrings
from .utils import from_pandas, to_pandas
if sys.version_info[0] == 3 and sys.version_info[1] >= 7:
# Python >= 3.7
from re import Pattern as _pattern_type
else:
# Python <= 3.6
from re import _pattern_type
@_inherit_docstrings(pandas.Series, excluded=[pandas.Series, pandas.Series.__init__])
class Series(BasePandasDataset):
def __init__(
self,
data=None,
index=None,
dtype=None,
name=None,
copy=False,
fastpath=False,
query_compiler=None,
):
"""Constructor for a Series object.
Args:
series_oids ([ObjectID]): The list of remote Series objects.
"""
if query_compiler is None: