diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/__init__.py b/src/pybind/mgr/rook/rook-client-python/rook_client/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3fa2272dd4e8b6e532e5c059cbde3eb8e15dd9de --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/__init__.py @@ -0,0 +1 @@ +from ._helper import STRICT \ No newline at end of file diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/_helper.py b/src/pybind/mgr/rook/rook-client-python/rook_client/_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..b4c7793f0840de5cd4dddbc83885fc8ee5bfe105 --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/_helper.py @@ -0,0 +1,108 @@ +import logging +try: + from typing import List, Dict, Any, Optional +except ImportError: + pass + +logger = logging.getLogger(__name__) + +# Tricking mypy to think `_omit`'s type is NoneType +# To make us not add things like `Union[Optional[str], OmitType]` +NoneType = type(None) +_omit = None # type: NoneType +_omit = object() # type: ignore + + +# Don't add any additionalProperties to objects. Useful for completeness testing +STRICT = False + + +def _property_from_json(data, breadcrumb, name, py_name, typ, required, nullable): + if not required and name not in data: + return _omit + obj = data[name] + if nullable and obj is None: + return obj + if issubclass(typ, CrdObject) or issubclass(typ, CrdObjectList): + return typ.from_json(obj, breadcrumb + '.' + name) + return obj + + +class CrdObject(object): + _properties = [] # type: List + + def __init__(self, **kwargs): + for prop in self._properties: + setattr(self, prop[1], kwargs.pop(prop[1])) + if kwargs: + raise TypeError( + '{} got unexpected arguments {}'.format(self.__class__.__name__, kwargs.keys())) + self._additionalProperties = {} # type: Dict[str, Any] + + def _property_impl(self, name): + obj = getattr(self, '_' + name) + if obj is _omit: + raise AttributeError(name + ' not found') + return obj + + def _property_to_json(self, name, py_name, typ, required, nullable): + obj = getattr(self, '_' + py_name) + if issubclass(typ, CrdObject) or issubclass(typ, CrdObjectList): + if nullable and obj is None: + return obj + if not required and obj is _omit: + return obj + return obj.to_json() + else: + return obj + + def to_json(self): + # type: () -> Dict[str, Any] + res = {p[0]: self._property_to_json(*p) for p in self._properties} + res.update(self._additionalProperties) + return {k: v for k, v in res.items() if v is not _omit} + + @classmethod + def from_json(cls, data, breadcrumb=''): + try: + sanitized = { + p[1]: _property_from_json(data, breadcrumb, *p) for p in cls._properties + } + extra = {k:v for k,v in data.items() if k not in sanitized} + ret = cls(**sanitized) + ret._additionalProperties = {} if STRICT else extra + return ret + except (TypeError, AttributeError, KeyError): + logger.exception(breadcrumb) + raise + + +class CrdClass(CrdObject): + @classmethod + def from_json(cls, data, breadcrumb=''): + kind = data['kind'] + if kind != cls.__name__: + raise ValueError("kind mismatch: {} != {}".format(kind, cls.__name__)) + return super(CrdClass, cls).from_json(data, breadcrumb) + + def to_json(self): + ret = super(CrdClass, self).to_json() + ret['kind'] = self.__class__.__name__ + return ret + + +class CrdObjectList(list): + # Py3: Replace `Any` with `TypeVar('T_CrdObject', bound='CrdObject')` + _items_type = None # type: Optional[Any] + + def to_json(self): + # type: () -> List + if self._items_type is None: + return self + return [e.to_json() for e in self] + + @classmethod + def from_json(cls, data, breadcrumb=''): + if cls._items_type is None: + return cls(data) + return cls(cls._items_type.from_json(e, breadcrumb + '[]') for e in data) diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/cassandra/__init__.py b/src/pybind/mgr/rook/rook-client-python/rook_client/cassandra/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/cassandra/cluster.py b/src/pybind/mgr/rook/rook-client-python/rook_client/cassandra/cluster.py new file mode 100644 index 0000000000000000000000000000000000000000..56dc8a8fe7aeeddf8334a4cc02d237288072a5ad --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/cassandra/cluster.py @@ -0,0 +1,308 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class Storage(CrdObject): + _properties = [ + ('volumeClaimTemplates', 'volumeClaimTemplates', object, True, False) + ] + + def __init__(self, + volumeClaimTemplates, # type: Any + ): + super(Storage, self).__init__( + volumeClaimTemplates=volumeClaimTemplates, + ) + + @property + def volumeClaimTemplates(self): + # type: () -> Any + return self._property_impl('volumeClaimTemplates') + + @volumeClaimTemplates.setter + def volumeClaimTemplates(self, new_val): + # type: (Any) -> None + self._volumeClaimTemplates = new_val + + +class Resources(CrdObject): + _properties = [ + ('cassandra', 'cassandra', object, False, False), + ('sidecar', 'sidecar', object, False, False) + ] + + def __init__(self, + cassandra=_omit, # type: Optional[Any] + sidecar=_omit, # type: Optional[Any] + ): + super(Resources, self).__init__( + cassandra=cassandra, + sidecar=sidecar, + ) + + @property + def cassandra(self): + # type: () -> Any + return self._property_impl('cassandra') + + @cassandra.setter + def cassandra(self, new_val): + # type: (Optional[Any]) -> None + self._cassandra = new_val + + @property + def sidecar(self): + # type: () -> Any + return self._property_impl('sidecar') + + @sidecar.setter + def sidecar(self, new_val): + # type: (Optional[Any]) -> None + self._sidecar = new_val + + +class RacksItem(CrdObject): + _properties = [ + ('name', 'name', str, True, False), + ('members', 'members', int, True, False), + ('configMapName', 'configMapName', str, False, False), + ('storage', 'storage', Storage, True, False), + ('placement', 'placement', object, False, False), + ('resources', 'resources', Resources, True, False), + ('sidecarImage', 'sidecarImage', object, False, False) + ] + + def __init__(self, + name, # type: str + members, # type: int + storage, # type: Storage + resources, # type: Resources + configMapName=_omit, # type: Optional[str] + placement=_omit, # type: Optional[Any] + sidecarImage=_omit, # type: Optional[Any] + ): + super(RacksItem, self).__init__( + name=name, + members=members, + storage=storage, + resources=resources, + configMapName=configMapName, + placement=placement, + sidecarImage=sidecarImage, + ) + + @property + def name(self): + # type: () -> str + return self._property_impl('name') + + @name.setter + def name(self, new_val): + # type: (str) -> None + self._name = new_val + + @property + def members(self): + # type: () -> int + return self._property_impl('members') + + @members.setter + def members(self, new_val): + # type: (int) -> None + self._members = new_val + + @property + def configMapName(self): + # type: () -> str + return self._property_impl('configMapName') + + @configMapName.setter + def configMapName(self, new_val): + # type: (Optional[str]) -> None + self._configMapName = new_val + + @property + def storage(self): + # type: () -> Storage + return self._property_impl('storage') + + @storage.setter + def storage(self, new_val): + # type: (Storage) -> None + self._storage = new_val + + @property + def placement(self): + # type: () -> Any + return self._property_impl('placement') + + @placement.setter + def placement(self, new_val): + # type: (Optional[Any]) -> None + self._placement = new_val + + @property + def resources(self): + # type: () -> Resources + return self._property_impl('resources') + + @resources.setter + def resources(self, new_val): + # type: (Resources) -> None + self._resources = new_val + + @property + def sidecarImage(self): + # type: () -> Any + return self._property_impl('sidecarImage') + + @sidecarImage.setter + def sidecarImage(self, new_val): + # type: (Optional[Any]) -> None + self._sidecarImage = new_val + + +class RacksList(CrdObjectList): + _items_type = RacksItem + + +class Datacenter(CrdObject): + _properties = [ + ('name', 'name', str, True, False), + ('racks', 'racks', RacksList, False, False) + ] + + def __init__(self, + name, # type: str + racks=_omit, # type: Optional[Union[List[RacksItem], CrdObjectList]] + ): + super(Datacenter, self).__init__( + name=name, + racks=racks, + ) + + @property + def name(self): + # type: () -> str + return self._property_impl('name') + + @name.setter + def name(self, new_val): + # type: (str) -> None + self._name = new_val + + @property + def racks(self): + # type: () -> Union[List[RacksItem], CrdObjectList] + return self._property_impl('racks') + + @racks.setter + def racks(self, new_val): + # type: (Optional[Union[List[RacksItem], CrdObjectList]]) -> None + self._racks = new_val + + +class Spec(CrdObject): + _properties = [ + ('version', 'version', str, True, False), + ('datacenter', 'datacenter', Datacenter, True, False) + ] + + def __init__(self, + version, # type: str + datacenter, # type: Datacenter + ): + super(Spec, self).__init__( + version=version, + datacenter=datacenter, + ) + + @property + def version(self): + # type: () -> str + return self._property_impl('version') + + @version.setter + def version(self, new_val): + # type: (str) -> None + self._version = new_val + + @property + def datacenter(self): + # type: () -> Datacenter + return self._property_impl('datacenter') + + @datacenter.setter + def datacenter(self, new_val): + # type: (Datacenter) -> None + self._datacenter = new_val + + +class Cluster(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(Cluster, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/__init__.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephclient.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephclient.py new file mode 100644 index 0000000000000000000000000000000000000000..52894eadf0414175e0a77b79e2014afe812dc36c --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephclient.py @@ -0,0 +1,95 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class Spec(CrdObject): + _properties = [ + ('caps', 'caps', object, False, False) + ] + + def __init__(self, + caps=_omit, # type: Optional[Any] + ): + super(Spec, self).__init__( + caps=caps, + ) + + @property + def caps(self): + # type: () -> Any + return self._property_impl('caps') + + @caps.setter + def caps(self, new_val): + # type: (Optional[Any]) -> None + self._caps = new_val + + +class CephClient(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(CephClient, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephcluster.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephcluster.py new file mode 100644 index 0000000000000000000000000000000000000000..d168c3c7c6d2c3863393579932eca1f5f5967772 --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephcluster.py @@ -0,0 +1,1119 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class CephVersion(CrdObject): + _properties = [ + ('allowUnsupported', 'allowUnsupported', bool, False, False), + ('image', 'image', str, False, False) + ] + + def __init__(self, + allowUnsupported=_omit, # type: Optional[bool] + image=_omit, # type: Optional[str] + ): + super(CephVersion, self).__init__( + allowUnsupported=allowUnsupported, + image=image, + ) + + @property + def allowUnsupported(self): + # type: () -> bool + return self._property_impl('allowUnsupported') + + @allowUnsupported.setter + def allowUnsupported(self, new_val): + # type: (Optional[bool]) -> None + self._allowUnsupported = new_val + + @property + def image(self): + # type: () -> str + return self._property_impl('image') + + @image.setter + def image(self, new_val): + # type: (Optional[str]) -> None + self._image = new_val + + +class Dashboard(CrdObject): + _properties = [ + ('enabled', 'enabled', bool, False, False), + ('urlPrefix', 'urlPrefix', str, False, False), + ('port', 'port', int, False, False), + ('ssl', 'ssl', bool, False, False) + ] + + def __init__(self, + enabled=_omit, # type: Optional[bool] + urlPrefix=_omit, # type: Optional[str] + port=_omit, # type: Optional[int] + ssl=_omit, # type: Optional[bool] + ): + super(Dashboard, self).__init__( + enabled=enabled, + urlPrefix=urlPrefix, + port=port, + ssl=ssl, + ) + + @property + def enabled(self): + # type: () -> bool + return self._property_impl('enabled') + + @enabled.setter + def enabled(self, new_val): + # type: (Optional[bool]) -> None + self._enabled = new_val + + @property + def urlPrefix(self): + # type: () -> str + return self._property_impl('urlPrefix') + + @urlPrefix.setter + def urlPrefix(self, new_val): + # type: (Optional[str]) -> None + self._urlPrefix = new_val + + @property + def port(self): + # type: () -> int + return self._property_impl('port') + + @port.setter + def port(self, new_val): + # type: (Optional[int]) -> None + self._port = new_val + + @property + def ssl(self): + # type: () -> bool + return self._property_impl('ssl') + + @ssl.setter + def ssl(self, new_val): + # type: (Optional[bool]) -> None + self._ssl = new_val + + +class DisruptionManagement(CrdObject): + _properties = [ + ('machineDisruptionBudgetNamespace', 'machineDisruptionBudgetNamespace', str, False, False), + ('managePodBudgets', 'managePodBudgets', bool, False, False), + ('osdMaintenanceTimeout', 'osdMaintenanceTimeout', int, False, False), + ('manageMachineDisruptionBudgets', 'manageMachineDisruptionBudgets', bool, False, False) + ] + + def __init__(self, + machineDisruptionBudgetNamespace=_omit, # type: Optional[str] + managePodBudgets=_omit, # type: Optional[bool] + osdMaintenanceTimeout=_omit, # type: Optional[int] + manageMachineDisruptionBudgets=_omit, # type: Optional[bool] + ): + super(DisruptionManagement, self).__init__( + machineDisruptionBudgetNamespace=machineDisruptionBudgetNamespace, + managePodBudgets=managePodBudgets, + osdMaintenanceTimeout=osdMaintenanceTimeout, + manageMachineDisruptionBudgets=manageMachineDisruptionBudgets, + ) + + @property + def machineDisruptionBudgetNamespace(self): + # type: () -> str + return self._property_impl('machineDisruptionBudgetNamespace') + + @machineDisruptionBudgetNamespace.setter + def machineDisruptionBudgetNamespace(self, new_val): + # type: (Optional[str]) -> None + self._machineDisruptionBudgetNamespace = new_val + + @property + def managePodBudgets(self): + # type: () -> bool + return self._property_impl('managePodBudgets') + + @managePodBudgets.setter + def managePodBudgets(self, new_val): + # type: (Optional[bool]) -> None + self._managePodBudgets = new_val + + @property + def osdMaintenanceTimeout(self): + # type: () -> int + return self._property_impl('osdMaintenanceTimeout') + + @osdMaintenanceTimeout.setter + def osdMaintenanceTimeout(self, new_val): + # type: (Optional[int]) -> None + self._osdMaintenanceTimeout = new_val + + @property + def manageMachineDisruptionBudgets(self): + # type: () -> bool + return self._property_impl('manageMachineDisruptionBudgets') + + @manageMachineDisruptionBudgets.setter + def manageMachineDisruptionBudgets(self, new_val): + # type: (Optional[bool]) -> None + self._manageMachineDisruptionBudgets = new_val + + +class Mon(CrdObject): + _properties = [ + ('allowMultiplePerNode', 'allowMultiplePerNode', bool, False, False), + ('count', 'count', int, False, False), + ('volumeClaimTemplate', 'volumeClaimTemplate', object, False, False) + ] + + def __init__(self, + allowMultiplePerNode=_omit, # type: Optional[bool] + count=_omit, # type: Optional[int] + volumeClaimTemplate=_omit, # type: Optional[Any] + ): + super(Mon, self).__init__( + allowMultiplePerNode=allowMultiplePerNode, + count=count, + volumeClaimTemplate=volumeClaimTemplate, + ) + + @property + def allowMultiplePerNode(self): + # type: () -> bool + return self._property_impl('allowMultiplePerNode') + + @allowMultiplePerNode.setter + def allowMultiplePerNode(self, new_val): + # type: (Optional[bool]) -> None + self._allowMultiplePerNode = new_val + + @property + def count(self): + # type: () -> int + return self._property_impl('count') + + @count.setter + def count(self, new_val): + # type: (Optional[int]) -> None + self._count = new_val + + @property + def volumeClaimTemplate(self): + # type: () -> Any + return self._property_impl('volumeClaimTemplate') + + @volumeClaimTemplate.setter + def volumeClaimTemplate(self, new_val): + # type: (Optional[Any]) -> None + self._volumeClaimTemplate = new_val + + +class ModulesItem(CrdObject): + _properties = [ + ('name', 'name', str, False, False), + ('enabled', 'enabled', bool, False, False) + ] + + def __init__(self, + name=_omit, # type: Optional[str] + enabled=_omit, # type: Optional[bool] + ): + super(ModulesItem, self).__init__( + name=name, + enabled=enabled, + ) + + @property + def name(self): + # type: () -> str + return self._property_impl('name') + + @name.setter + def name(self, new_val): + # type: (Optional[str]) -> None + self._name = new_val + + @property + def enabled(self): + # type: () -> bool + return self._property_impl('enabled') + + @enabled.setter + def enabled(self, new_val): + # type: (Optional[bool]) -> None + self._enabled = new_val + + +class ModulesList(CrdObjectList): + _items_type = ModulesItem + + +class Mgr(CrdObject): + _properties = [ + ('modules', 'modules', ModulesList, False, False) + ] + + def __init__(self, + modules=_omit, # type: Optional[Union[List[ModulesItem], CrdObjectList]] + ): + super(Mgr, self).__init__( + modules=modules, + ) + + @property + def modules(self): + # type: () -> Union[List[ModulesItem], CrdObjectList] + return self._property_impl('modules') + + @modules.setter + def modules(self, new_val): + # type: (Optional[Union[List[ModulesItem], CrdObjectList]]) -> None + self._modules = new_val + + +class Network(CrdObject): + _properties = [ + ('hostNetwork', 'hostNetwork', bool, False, False), + ('provider', 'provider', str, False, False), + ('selectors', 'selectors', object, False, False) + ] + + def __init__(self, + hostNetwork=_omit, # type: Optional[bool] + provider=_omit, # type: Optional[str] + selectors=_omit, # type: Optional[Any] + ): + super(Network, self).__init__( + hostNetwork=hostNetwork, + provider=provider, + selectors=selectors, + ) + + @property + def hostNetwork(self): + # type: () -> bool + return self._property_impl('hostNetwork') + + @hostNetwork.setter + def hostNetwork(self, new_val): + # type: (Optional[bool]) -> None + self._hostNetwork = new_val + + @property + def provider(self): + # type: () -> str + return self._property_impl('provider') + + @provider.setter + def provider(self, new_val): + # type: (Optional[str]) -> None + self._provider = new_val + + @property + def selectors(self): + # type: () -> Any + return self._property_impl('selectors') + + @selectors.setter + def selectors(self, new_val): + # type: (Optional[Any]) -> None + self._selectors = new_val + + +class Config(CrdObject): + _properties = [ + ('metadataDevice', 'metadataDevice', str, False, False), + ('storeType', 'storeType', str, False, False), + ('databaseSizeMB', 'databaseSizeMB', str, False, False), + ('walSizeMB', 'walSizeMB', str, False, False), + ('journalSizeMB', 'journalSizeMB', str, False, False), + ('osdsPerDevice', 'osdsPerDevice', str, False, False), + ('encryptedDevice', 'encryptedDevice', str, False, False) + ] + + def __init__(self, + metadataDevice=_omit, # type: Optional[str] + storeType=_omit, # type: Optional[str] + databaseSizeMB=_omit, # type: Optional[str] + walSizeMB=_omit, # type: Optional[str] + journalSizeMB=_omit, # type: Optional[str] + osdsPerDevice=_omit, # type: Optional[str] + encryptedDevice=_omit, # type: Optional[str] + ): + super(Config, self).__init__( + metadataDevice=metadataDevice, + storeType=storeType, + databaseSizeMB=databaseSizeMB, + walSizeMB=walSizeMB, + journalSizeMB=journalSizeMB, + osdsPerDevice=osdsPerDevice, + encryptedDevice=encryptedDevice, + ) + + @property + def metadataDevice(self): + # type: () -> str + return self._property_impl('metadataDevice') + + @metadataDevice.setter + def metadataDevice(self, new_val): + # type: (Optional[str]) -> None + self._metadataDevice = new_val + + @property + def storeType(self): + # type: () -> str + return self._property_impl('storeType') + + @storeType.setter + def storeType(self, new_val): + # type: (Optional[str]) -> None + self._storeType = new_val + + @property + def databaseSizeMB(self): + # type: () -> str + return self._property_impl('databaseSizeMB') + + @databaseSizeMB.setter + def databaseSizeMB(self, new_val): + # type: (Optional[str]) -> None + self._databaseSizeMB = new_val + + @property + def walSizeMB(self): + # type: () -> str + return self._property_impl('walSizeMB') + + @walSizeMB.setter + def walSizeMB(self, new_val): + # type: (Optional[str]) -> None + self._walSizeMB = new_val + + @property + def journalSizeMB(self): + # type: () -> str + return self._property_impl('journalSizeMB') + + @journalSizeMB.setter + def journalSizeMB(self, new_val): + # type: (Optional[str]) -> None + self._journalSizeMB = new_val + + @property + def osdsPerDevice(self): + # type: () -> str + return self._property_impl('osdsPerDevice') + + @osdsPerDevice.setter + def osdsPerDevice(self, new_val): + # type: (Optional[str]) -> None + self._osdsPerDevice = new_val + + @property + def encryptedDevice(self): + # type: () -> str + return self._property_impl('encryptedDevice') + + @encryptedDevice.setter + def encryptedDevice(self, new_val): + # type: (Optional[str]) -> None + self._encryptedDevice = new_val + + +class DirectoriesItem(CrdObject): + _properties = [ + ('path', 'path', str, False, False) + ] + + def __init__(self, + path=_omit, # type: Optional[str] + ): + super(DirectoriesItem, self).__init__( + path=path, + ) + + @property + def path(self): + # type: () -> str + return self._property_impl('path') + + @path.setter + def path(self, new_val): + # type: (Optional[str]) -> None + self._path = new_val + + +class DirectoriesList(CrdObjectList): + _items_type = DirectoriesItem + + +class DevicesItem(CrdObject): + _properties = [ + ('name', 'name', str, False, False), + ('config', 'config', object, False, False) + ] + + def __init__(self, + name=_omit, # type: Optional[str] + config=_omit, # type: Optional[Any] + ): + super(DevicesItem, self).__init__( + name=name, + config=config, + ) + + @property + def name(self): + # type: () -> str + return self._property_impl('name') + + @name.setter + def name(self, new_val): + # type: (Optional[str]) -> None + self._name = new_val + + @property + def config(self): + # type: () -> Any + return self._property_impl('config') + + @config.setter + def config(self, new_val): + # type: (Optional[Any]) -> None + self._config = new_val + + +class DevicesList(CrdObjectList): + _items_type = DevicesItem + + +class NodesItem(CrdObject): + _properties = [ + ('name', 'name', str, False, False), + ('config', 'config', Config, False, False), + ('useAllDevices', 'useAllDevices', bool, False, False), + ('deviceFilter', 'deviceFilter', str, False, False), + ('devicePathFilter', 'devicePathFilter', str, False, False), + ('directories', 'directories', DirectoriesList, False, False), + ('devices', 'devices', DevicesList, False, False), + ('resources', 'resources', object, False, False) + ] + + def __init__(self, + name=_omit, # type: Optional[str] + config=_omit, # type: Optional[Config] + useAllDevices=_omit, # type: Optional[bool] + deviceFilter=_omit, # type: Optional[str] + devicePathFilter=_omit, # type: Optional[str] + directories=_omit, # type: Optional[Union[List[DirectoriesItem], CrdObjectList]] + devices=_omit, # type: Optional[Union[List[DevicesItem], CrdObjectList]] + resources=_omit, # type: Optional[Any] + ): + super(NodesItem, self).__init__( + name=name, + config=config, + useAllDevices=useAllDevices, + deviceFilter=deviceFilter, + devicePathFilter=devicePathFilter, + directories=directories, + devices=devices, + resources=resources, + ) + + @property + def name(self): + # type: () -> str + return self._property_impl('name') + + @name.setter + def name(self, new_val): + # type: (Optional[str]) -> None + self._name = new_val + + @property + def config(self): + # type: () -> Config + return self._property_impl('config') + + @config.setter + def config(self, new_val): + # type: (Optional[Config]) -> None + self._config = new_val + + @property + def useAllDevices(self): + # type: () -> bool + return self._property_impl('useAllDevices') + + @useAllDevices.setter + def useAllDevices(self, new_val): + # type: (Optional[bool]) -> None + self._useAllDevices = new_val + + @property + def deviceFilter(self): + # type: () -> str + return self._property_impl('deviceFilter') + + @deviceFilter.setter + def deviceFilter(self, new_val): + # type: (Optional[str]) -> None + self._deviceFilter = new_val + + @property + def devicePathFilter(self): + # type: () -> str + return self._property_impl('devicePathFilter') + + @devicePathFilter.setter + def devicePathFilter(self, new_val): + # type: (Optional[str]) -> None + self._devicePathFilter = new_val + + @property + def directories(self): + # type: () -> Union[List[DirectoriesItem], CrdObjectList] + return self._property_impl('directories') + + @directories.setter + def directories(self, new_val): + # type: (Optional[Union[List[DirectoriesItem], CrdObjectList]]) -> None + self._directories = new_val + + @property + def devices(self): + # type: () -> Union[List[DevicesItem], CrdObjectList] + return self._property_impl('devices') + + @devices.setter + def devices(self, new_val): + # type: (Optional[Union[List[DevicesItem], CrdObjectList]]) -> None + self._devices = new_val + + @property + def resources(self): + # type: () -> Any + return self._property_impl('resources') + + @resources.setter + def resources(self, new_val): + # type: (Optional[Any]) -> None + self._resources = new_val + + +class NodesList(CrdObjectList): + _items_type = NodesItem + + +class Storage(CrdObject): + _properties = [ + ('disruptionManagement', 'disruptionManagement', DisruptionManagement, False, False), + ('useAllNodes', 'useAllNodes', bool, False, False), + ('nodes', 'nodes', NodesList, False, False), + ('useAllDevices', 'useAllDevices', bool, False, False), + ('deviceFilter', 'deviceFilter', str, False, False), + ('devicePathFilter', 'devicePathFilter', str, False, False), + ('directories', 'directories', DirectoriesList, False, False), + ('config', 'config', object, False, False), + ('storageClassDeviceSets', 'storageClassDeviceSets', object, False, False) + ] + + def __init__(self, + disruptionManagement=_omit, # type: Optional[DisruptionManagement] + useAllNodes=_omit, # type: Optional[bool] + nodes=_omit, # type: Optional[Union[List[NodesItem], CrdObjectList]] + useAllDevices=_omit, # type: Optional[bool] + deviceFilter=_omit, # type: Optional[str] + devicePathFilter=_omit, # type: Optional[str] + directories=_omit, # type: Optional[Union[List[DirectoriesItem], CrdObjectList]] + config=_omit, # type: Optional[Any] + storageClassDeviceSets=_omit, # type: Optional[Any] + ): + super(Storage, self).__init__( + disruptionManagement=disruptionManagement, + useAllNodes=useAllNodes, + nodes=nodes, + useAllDevices=useAllDevices, + deviceFilter=deviceFilter, + devicePathFilter=devicePathFilter, + directories=directories, + config=config, + storageClassDeviceSets=storageClassDeviceSets, + ) + + @property + def disruptionManagement(self): + # type: () -> DisruptionManagement + return self._property_impl('disruptionManagement') + + @disruptionManagement.setter + def disruptionManagement(self, new_val): + # type: (Optional[DisruptionManagement]) -> None + self._disruptionManagement = new_val + + @property + def useAllNodes(self): + # type: () -> bool + return self._property_impl('useAllNodes') + + @useAllNodes.setter + def useAllNodes(self, new_val): + # type: (Optional[bool]) -> None + self._useAllNodes = new_val + + @property + def nodes(self): + # type: () -> Union[List[NodesItem], CrdObjectList] + return self._property_impl('nodes') + + @nodes.setter + def nodes(self, new_val): + # type: (Optional[Union[List[NodesItem], CrdObjectList]]) -> None + self._nodes = new_val + + @property + def useAllDevices(self): + # type: () -> bool + return self._property_impl('useAllDevices') + + @useAllDevices.setter + def useAllDevices(self, new_val): + # type: (Optional[bool]) -> None + self._useAllDevices = new_val + + @property + def deviceFilter(self): + # type: () -> str + return self._property_impl('deviceFilter') + + @deviceFilter.setter + def deviceFilter(self, new_val): + # type: (Optional[str]) -> None + self._deviceFilter = new_val + + @property + def devicePathFilter(self): + # type: () -> str + return self._property_impl('devicePathFilter') + + @devicePathFilter.setter + def devicePathFilter(self, new_val): + # type: (Optional[str]) -> None + self._devicePathFilter = new_val + + @property + def directories(self): + # type: () -> Union[List[DirectoriesItem], CrdObjectList] + return self._property_impl('directories') + + @directories.setter + def directories(self, new_val): + # type: (Optional[Union[List[DirectoriesItem], CrdObjectList]]) -> None + self._directories = new_val + + @property + def config(self): + # type: () -> Any + return self._property_impl('config') + + @config.setter + def config(self, new_val): + # type: (Optional[Any]) -> None + self._config = new_val + + @property + def storageClassDeviceSets(self): + # type: () -> Any + return self._property_impl('storageClassDeviceSets') + + @storageClassDeviceSets.setter + def storageClassDeviceSets(self, new_val): + # type: (Optional[Any]) -> None + self._storageClassDeviceSets = new_val + + +class Monitoring(CrdObject): + _properties = [ + ('enabled', 'enabled', bool, False, False), + ('rulesNamespace', 'rulesNamespace', str, False, False) + ] + + def __init__(self, + enabled=_omit, # type: Optional[bool] + rulesNamespace=_omit, # type: Optional[str] + ): + super(Monitoring, self).__init__( + enabled=enabled, + rulesNamespace=rulesNamespace, + ) + + @property + def enabled(self): + # type: () -> bool + return self._property_impl('enabled') + + @enabled.setter + def enabled(self, new_val): + # type: (Optional[bool]) -> None + self._enabled = new_val + + @property + def rulesNamespace(self): + # type: () -> str + return self._property_impl('rulesNamespace') + + @rulesNamespace.setter + def rulesNamespace(self, new_val): + # type: (Optional[str]) -> None + self._rulesNamespace = new_val + + +class RbdMirroring(CrdObject): + _properties = [ + ('workers', 'workers', int, False, False) + ] + + def __init__(self, + workers=_omit, # type: Optional[int] + ): + super(RbdMirroring, self).__init__( + workers=workers, + ) + + @property + def workers(self): + # type: () -> int + return self._property_impl('workers') + + @workers.setter + def workers(self, new_val): + # type: (Optional[int]) -> None + self._workers = new_val + + +class External(CrdObject): + _properties = [ + ('enable', 'enable', bool, False, False) + ] + + def __init__(self, + enable=_omit, # type: Optional[bool] + ): + super(External, self).__init__( + enable=enable, + ) + + @property + def enable(self): + # type: () -> bool + return self._property_impl('enable') + + @enable.setter + def enable(self, new_val): + # type: (Optional[bool]) -> None + self._enable = new_val + + +class Spec(CrdObject): + _properties = [ + ('annotations', 'annotations', object, False, False), + ('cephVersion', 'cephVersion', CephVersion, False, False), + ('dashboard', 'dashboard', Dashboard, False, False), + ('dataDirHostPath', 'dataDirHostPath', str, False, False), + ('disruptionManagement', 'disruptionManagement', DisruptionManagement, False, False), + ('skipUpgradeChecks', 'skipUpgradeChecks', bool, False, False), + ('continueUpgradeAfterChecksEvenIfNotHealthy', 'continueUpgradeAfterChecksEvenIfNotHealthy', bool, False, False), + ('mon', 'mon', Mon, False, False), + ('mgr', 'mgr', Mgr, False, False), + ('network', 'network', Network, False, False), + ('storage', 'storage', Storage, False, False), + ('monitoring', 'monitoring', Monitoring, False, False), + ('rbdMirroring', 'rbdMirroring', RbdMirroring, False, False), + ('removeOSDsIfOutAndSafeToRemove', 'removeOSDsIfOutAndSafeToRemove', bool, False, False), + ('external', 'external', External, False, False), + ('placement', 'placement', object, False, False), + ('resources', 'resources', object, False, False) + ] + + def __init__(self, + annotations=_omit, # type: Optional[Any] + cephVersion=_omit, # type: Optional[CephVersion] + dashboard=_omit, # type: Optional[Dashboard] + dataDirHostPath=_omit, # type: Optional[str] + disruptionManagement=_omit, # type: Optional[DisruptionManagement] + skipUpgradeChecks=_omit, # type: Optional[bool] + continueUpgradeAfterChecksEvenIfNotHealthy=_omit, # type: Optional[bool] + mon=_omit, # type: Optional[Mon] + mgr=_omit, # type: Optional[Mgr] + network=_omit, # type: Optional[Network] + storage=_omit, # type: Optional[Storage] + monitoring=_omit, # type: Optional[Monitoring] + rbdMirroring=_omit, # type: Optional[RbdMirroring] + removeOSDsIfOutAndSafeToRemove=_omit, # type: Optional[bool] + external=_omit, # type: Optional[External] + placement=_omit, # type: Optional[Any] + resources=_omit, # type: Optional[Any] + ): + super(Spec, self).__init__( + annotations=annotations, + cephVersion=cephVersion, + dashboard=dashboard, + dataDirHostPath=dataDirHostPath, + disruptionManagement=disruptionManagement, + skipUpgradeChecks=skipUpgradeChecks, + continueUpgradeAfterChecksEvenIfNotHealthy=continueUpgradeAfterChecksEvenIfNotHealthy, + mon=mon, + mgr=mgr, + network=network, + storage=storage, + monitoring=monitoring, + rbdMirroring=rbdMirroring, + removeOSDsIfOutAndSafeToRemove=removeOSDsIfOutAndSafeToRemove, + external=external, + placement=placement, + resources=resources, + ) + + @property + def annotations(self): + # type: () -> Any + return self._property_impl('annotations') + + @annotations.setter + def annotations(self, new_val): + # type: (Optional[Any]) -> None + self._annotations = new_val + + @property + def cephVersion(self): + # type: () -> CephVersion + return self._property_impl('cephVersion') + + @cephVersion.setter + def cephVersion(self, new_val): + # type: (Optional[CephVersion]) -> None + self._cephVersion = new_val + + @property + def dashboard(self): + # type: () -> Dashboard + return self._property_impl('dashboard') + + @dashboard.setter + def dashboard(self, new_val): + # type: (Optional[Dashboard]) -> None + self._dashboard = new_val + + @property + def dataDirHostPath(self): + # type: () -> str + return self._property_impl('dataDirHostPath') + + @dataDirHostPath.setter + def dataDirHostPath(self, new_val): + # type: (Optional[str]) -> None + self._dataDirHostPath = new_val + + @property + def disruptionManagement(self): + # type: () -> DisruptionManagement + return self._property_impl('disruptionManagement') + + @disruptionManagement.setter + def disruptionManagement(self, new_val): + # type: (Optional[DisruptionManagement]) -> None + self._disruptionManagement = new_val + + @property + def skipUpgradeChecks(self): + # type: () -> bool + return self._property_impl('skipUpgradeChecks') + + @skipUpgradeChecks.setter + def skipUpgradeChecks(self, new_val): + # type: (Optional[bool]) -> None + self._skipUpgradeChecks = new_val + + @property + def continueUpgradeAfterChecksEvenIfNotHealthy(self): + # type: () -> bool + return self._property_impl('continueUpgradeAfterChecksEvenIfNotHealthy') + + @continueUpgradeAfterChecksEvenIfNotHealthy.setter + def continueUpgradeAfterChecksEvenIfNotHealthy(self, new_val): + # type: (Optional[bool]) -> None + self._continueUpgradeAfterChecksEvenIfNotHealthy = new_val + + @property + def mon(self): + # type: () -> Mon + return self._property_impl('mon') + + @mon.setter + def mon(self, new_val): + # type: (Optional[Mon]) -> None + self._mon = new_val + + @property + def mgr(self): + # type: () -> Mgr + return self._property_impl('mgr') + + @mgr.setter + def mgr(self, new_val): + # type: (Optional[Mgr]) -> None + self._mgr = new_val + + @property + def network(self): + # type: () -> Network + return self._property_impl('network') + + @network.setter + def network(self, new_val): + # type: (Optional[Network]) -> None + self._network = new_val + + @property + def storage(self): + # type: () -> Storage + return self._property_impl('storage') + + @storage.setter + def storage(self, new_val): + # type: (Optional[Storage]) -> None + self._storage = new_val + + @property + def monitoring(self): + # type: () -> Monitoring + return self._property_impl('monitoring') + + @monitoring.setter + def monitoring(self, new_val): + # type: (Optional[Monitoring]) -> None + self._monitoring = new_val + + @property + def rbdMirroring(self): + # type: () -> RbdMirroring + return self._property_impl('rbdMirroring') + + @rbdMirroring.setter + def rbdMirroring(self, new_val): + # type: (Optional[RbdMirroring]) -> None + self._rbdMirroring = new_val + + @property + def removeOSDsIfOutAndSafeToRemove(self): + # type: () -> bool + return self._property_impl('removeOSDsIfOutAndSafeToRemove') + + @removeOSDsIfOutAndSafeToRemove.setter + def removeOSDsIfOutAndSafeToRemove(self, new_val): + # type: (Optional[bool]) -> None + self._removeOSDsIfOutAndSafeToRemove = new_val + + @property + def external(self): + # type: () -> External + return self._property_impl('external') + + @external.setter + def external(self, new_val): + # type: (Optional[External]) -> None + self._external = new_val + + @property + def placement(self): + # type: () -> Any + return self._property_impl('placement') + + @placement.setter + def placement(self, new_val): + # type: (Optional[Any]) -> None + self._placement = new_val + + @property + def resources(self): + # type: () -> Any + return self._property_impl('resources') + + @resources.setter + def resources(self, new_val): + # type: (Optional[Any]) -> None + self._resources = new_val + + +class CephCluster(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(CephCluster, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephfilesystem.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephfilesystem.py new file mode 100644 index 0000000000000000000000000000000000000000..ac217711d71f9218dd66aa2af5f34854f3fc6efc --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephfilesystem.py @@ -0,0 +1,370 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class MetadataServer(CrdObject): + _properties = [ + ('activeCount', 'activeCount', int, False, False), + ('activeStandby', 'activeStandby', bool, False, False), + ('annotations', 'annotations', object, False, False), + ('placement', 'placement', object, False, False), + ('resources', 'resources', object, False, False) + ] + + def __init__(self, + activeCount=_omit, # type: Optional[int] + activeStandby=_omit, # type: Optional[bool] + annotations=_omit, # type: Optional[Any] + placement=_omit, # type: Optional[Any] + resources=_omit, # type: Optional[Any] + ): + super(MetadataServer, self).__init__( + activeCount=activeCount, + activeStandby=activeStandby, + annotations=annotations, + placement=placement, + resources=resources, + ) + + @property + def activeCount(self): + # type: () -> int + return self._property_impl('activeCount') + + @activeCount.setter + def activeCount(self, new_val): + # type: (Optional[int]) -> None + self._activeCount = new_val + + @property + def activeStandby(self): + # type: () -> bool + return self._property_impl('activeStandby') + + @activeStandby.setter + def activeStandby(self, new_val): + # type: (Optional[bool]) -> None + self._activeStandby = new_val + + @property + def annotations(self): + # type: () -> Any + return self._property_impl('annotations') + + @annotations.setter + def annotations(self, new_val): + # type: (Optional[Any]) -> None + self._annotations = new_val + + @property + def placement(self): + # type: () -> Any + return self._property_impl('placement') + + @placement.setter + def placement(self, new_val): + # type: (Optional[Any]) -> None + self._placement = new_val + + @property + def resources(self): + # type: () -> Any + return self._property_impl('resources') + + @resources.setter + def resources(self, new_val): + # type: (Optional[Any]) -> None + self._resources = new_val + + +class Replicated(CrdObject): + _properties = [ + ('size', 'size', int, False, False) + ] + + def __init__(self, + size=_omit, # type: Optional[int] + ): + super(Replicated, self).__init__( + size=size, + ) + + @property + def size(self): + # type: () -> int + return self._property_impl('size') + + @size.setter + def size(self, new_val): + # type: (Optional[int]) -> None + self._size = new_val + + +class ErasureCoded(CrdObject): + _properties = [ + ('dataChunks', 'dataChunks', int, False, False), + ('codingChunks', 'codingChunks', int, False, False) + ] + + def __init__(self, + dataChunks=_omit, # type: Optional[int] + codingChunks=_omit, # type: Optional[int] + ): + super(ErasureCoded, self).__init__( + dataChunks=dataChunks, + codingChunks=codingChunks, + ) + + @property + def dataChunks(self): + # type: () -> int + return self._property_impl('dataChunks') + + @dataChunks.setter + def dataChunks(self, new_val): + # type: (Optional[int]) -> None + self._dataChunks = new_val + + @property + def codingChunks(self): + # type: () -> int + return self._property_impl('codingChunks') + + @codingChunks.setter + def codingChunks(self, new_val): + # type: (Optional[int]) -> None + self._codingChunks = new_val + + +class MetadataPool(CrdObject): + _properties = [ + ('failureDomain', 'failureDomain', str, False, False), + ('replicated', 'replicated', Replicated, False, False), + ('erasureCoded', 'erasureCoded', ErasureCoded, False, False) + ] + + def __init__(self, + failureDomain=_omit, # type: Optional[str] + replicated=_omit, # type: Optional[Replicated] + erasureCoded=_omit, # type: Optional[ErasureCoded] + ): + super(MetadataPool, self).__init__( + failureDomain=failureDomain, + replicated=replicated, + erasureCoded=erasureCoded, + ) + + @property + def failureDomain(self): + # type: () -> str + return self._property_impl('failureDomain') + + @failureDomain.setter + def failureDomain(self, new_val): + # type: (Optional[str]) -> None + self._failureDomain = new_val + + @property + def replicated(self): + # type: () -> Replicated + return self._property_impl('replicated') + + @replicated.setter + def replicated(self, new_val): + # type: (Optional[Replicated]) -> None + self._replicated = new_val + + @property + def erasureCoded(self): + # type: () -> ErasureCoded + return self._property_impl('erasureCoded') + + @erasureCoded.setter + def erasureCoded(self, new_val): + # type: (Optional[ErasureCoded]) -> None + self._erasureCoded = new_val + + +class DataPoolsItem(CrdObject): + _properties = [ + ('failureDomain', 'failureDomain', str, False, False), + ('replicated', 'replicated', Replicated, False, False), + ('erasureCoded', 'erasureCoded', ErasureCoded, False, False) + ] + + def __init__(self, + failureDomain=_omit, # type: Optional[str] + replicated=_omit, # type: Optional[Replicated] + erasureCoded=_omit, # type: Optional[ErasureCoded] + ): + super(DataPoolsItem, self).__init__( + failureDomain=failureDomain, + replicated=replicated, + erasureCoded=erasureCoded, + ) + + @property + def failureDomain(self): + # type: () -> str + return self._property_impl('failureDomain') + + @failureDomain.setter + def failureDomain(self, new_val): + # type: (Optional[str]) -> None + self._failureDomain = new_val + + @property + def replicated(self): + # type: () -> Replicated + return self._property_impl('replicated') + + @replicated.setter + def replicated(self, new_val): + # type: (Optional[Replicated]) -> None + self._replicated = new_val + + @property + def erasureCoded(self): + # type: () -> ErasureCoded + return self._property_impl('erasureCoded') + + @erasureCoded.setter + def erasureCoded(self, new_val): + # type: (Optional[ErasureCoded]) -> None + self._erasureCoded = new_val + + +class DataPoolsList(CrdObjectList): + _items_type = DataPoolsItem + + +class Spec(CrdObject): + _properties = [ + ('metadataServer', 'metadataServer', MetadataServer, False, False), + ('metadataPool', 'metadataPool', MetadataPool, False, False), + ('dataPools', 'dataPools', DataPoolsList, False, False), + ('preservePoolsOnDelete', 'preservePoolsOnDelete', bool, False, False) + ] + + def __init__(self, + metadataServer=_omit, # type: Optional[MetadataServer] + metadataPool=_omit, # type: Optional[MetadataPool] + dataPools=_omit, # type: Optional[Union[List[DataPoolsItem], CrdObjectList]] + preservePoolsOnDelete=_omit, # type: Optional[bool] + ): + super(Spec, self).__init__( + metadataServer=metadataServer, + metadataPool=metadataPool, + dataPools=dataPools, + preservePoolsOnDelete=preservePoolsOnDelete, + ) + + @property + def metadataServer(self): + # type: () -> MetadataServer + return self._property_impl('metadataServer') + + @metadataServer.setter + def metadataServer(self, new_val): + # type: (Optional[MetadataServer]) -> None + self._metadataServer = new_val + + @property + def metadataPool(self): + # type: () -> MetadataPool + return self._property_impl('metadataPool') + + @metadataPool.setter + def metadataPool(self, new_val): + # type: (Optional[MetadataPool]) -> None + self._metadataPool = new_val + + @property + def dataPools(self): + # type: () -> Union[List[DataPoolsItem], CrdObjectList] + return self._property_impl('dataPools') + + @dataPools.setter + def dataPools(self, new_val): + # type: (Optional[Union[List[DataPoolsItem], CrdObjectList]]) -> None + self._dataPools = new_val + + @property + def preservePoolsOnDelete(self): + # type: () -> bool + return self._property_impl('preservePoolsOnDelete') + + @preservePoolsOnDelete.setter + def preservePoolsOnDelete(self, new_val): + # type: (Optional[bool]) -> None + self._preservePoolsOnDelete = new_val + + +class CephFilesystem(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(CephFilesystem, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephnfs.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephnfs.py new file mode 100644 index 0000000000000000000000000000000000000000..c46533ec941854b3c78f1dcd37d26fa754d92bbb --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephnfs.py @@ -0,0 +1,206 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class Rados(CrdObject): + _properties = [ + ('pool', 'pool', str, False, False), + ('namespace', 'namespace', str, False, False) + ] + + def __init__(self, + pool=_omit, # type: Optional[str] + namespace=_omit, # type: Optional[str] + ): + super(Rados, self).__init__( + pool=pool, + namespace=namespace, + ) + + @property + def pool(self): + # type: () -> str + return self._property_impl('pool') + + @pool.setter + def pool(self, new_val): + # type: (Optional[str]) -> None + self._pool = new_val + + @property + def namespace(self): + # type: () -> str + return self._property_impl('namespace') + + @namespace.setter + def namespace(self, new_val): + # type: (Optional[str]) -> None + self._namespace = new_val + + +class Server(CrdObject): + _properties = [ + ('active', 'active', int, False, False), + ('annotations', 'annotations', object, False, False), + ('placement', 'placement', object, False, False), + ('resources', 'resources', object, False, False) + ] + + def __init__(self, + active=_omit, # type: Optional[int] + annotations=_omit, # type: Optional[Any] + placement=_omit, # type: Optional[Any] + resources=_omit, # type: Optional[Any] + ): + super(Server, self).__init__( + active=active, + annotations=annotations, + placement=placement, + resources=resources, + ) + + @property + def active(self): + # type: () -> int + return self._property_impl('active') + + @active.setter + def active(self, new_val): + # type: (Optional[int]) -> None + self._active = new_val + + @property + def annotations(self): + # type: () -> Any + return self._property_impl('annotations') + + @annotations.setter + def annotations(self, new_val): + # type: (Optional[Any]) -> None + self._annotations = new_val + + @property + def placement(self): + # type: () -> Any + return self._property_impl('placement') + + @placement.setter + def placement(self, new_val): + # type: (Optional[Any]) -> None + self._placement = new_val + + @property + def resources(self): + # type: () -> Any + return self._property_impl('resources') + + @resources.setter + def resources(self, new_val): + # type: (Optional[Any]) -> None + self._resources = new_val + + +class Spec(CrdObject): + _properties = [ + ('rados', 'rados', Rados, False, False), + ('server', 'server', Server, False, False) + ] + + def __init__(self, + rados=_omit, # type: Optional[Rados] + server=_omit, # type: Optional[Server] + ): + super(Spec, self).__init__( + rados=rados, + server=server, + ) + + @property + def rados(self): + # type: () -> Rados + return self._property_impl('rados') + + @rados.setter + def rados(self, new_val): + # type: (Optional[Rados]) -> None + self._rados = new_val + + @property + def server(self): + # type: () -> Server + return self._property_impl('server') + + @server.setter + def server(self, new_val): + # type: (Optional[Server]) -> None + self._server = new_val + + +class CephNFS(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(CephNFS, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephobjectstore.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephobjectstore.py new file mode 100644 index 0000000000000000000000000000000000000000..025080929e252089ae1f102e76f7c024ab90de46 --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephobjectstore.py @@ -0,0 +1,405 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class Gateway(CrdObject): + _properties = [ + ('type', 'type', str, False, False), + ('sslCertificateRef', 'sslCertificateRef', object, False, False), + ('port', 'port', int, False, False), + ('securePort', 'securePort', object, False, False), + ('instances', 'instances', int, False, False), + ('annotations', 'annotations', object, False, False), + ('placement', 'placement', object, False, False), + ('resources', 'resources', object, False, False) + ] + + def __init__(self, + type=_omit, # type: Optional[str] + sslCertificateRef=_omit, # type: Optional[Any] + port=_omit, # type: Optional[int] + securePort=_omit, # type: Optional[Any] + instances=_omit, # type: Optional[int] + annotations=_omit, # type: Optional[Any] + placement=_omit, # type: Optional[Any] + resources=_omit, # type: Optional[Any] + ): + super(Gateway, self).__init__( + type=type, + sslCertificateRef=sslCertificateRef, + port=port, + securePort=securePort, + instances=instances, + annotations=annotations, + placement=placement, + resources=resources, + ) + + @property + def type(self): + # type: () -> str + return self._property_impl('type') + + @type.setter + def type(self, new_val): + # type: (Optional[str]) -> None + self._type = new_val + + @property + def sslCertificateRef(self): + # type: () -> Any + return self._property_impl('sslCertificateRef') + + @sslCertificateRef.setter + def sslCertificateRef(self, new_val): + # type: (Optional[Any]) -> None + self._sslCertificateRef = new_val + + @property + def port(self): + # type: () -> int + return self._property_impl('port') + + @port.setter + def port(self, new_val): + # type: (Optional[int]) -> None + self._port = new_val + + @property + def securePort(self): + # type: () -> Any + return self._property_impl('securePort') + + @securePort.setter + def securePort(self, new_val): + # type: (Optional[Any]) -> None + self._securePort = new_val + + @property + def instances(self): + # type: () -> int + return self._property_impl('instances') + + @instances.setter + def instances(self, new_val): + # type: (Optional[int]) -> None + self._instances = new_val + + @property + def annotations(self): + # type: () -> Any + return self._property_impl('annotations') + + @annotations.setter + def annotations(self, new_val): + # type: (Optional[Any]) -> None + self._annotations = new_val + + @property + def placement(self): + # type: () -> Any + return self._property_impl('placement') + + @placement.setter + def placement(self, new_val): + # type: (Optional[Any]) -> None + self._placement = new_val + + @property + def resources(self): + # type: () -> Any + return self._property_impl('resources') + + @resources.setter + def resources(self, new_val): + # type: (Optional[Any]) -> None + self._resources = new_val + + +class Replicated(CrdObject): + _properties = [ + ('size', 'size', int, False, False) + ] + + def __init__(self, + size=_omit, # type: Optional[int] + ): + super(Replicated, self).__init__( + size=size, + ) + + @property + def size(self): + # type: () -> int + return self._property_impl('size') + + @size.setter + def size(self, new_val): + # type: (Optional[int]) -> None + self._size = new_val + + +class ErasureCoded(CrdObject): + _properties = [ + ('dataChunks', 'dataChunks', int, False, False), + ('codingChunks', 'codingChunks', int, False, False) + ] + + def __init__(self, + dataChunks=_omit, # type: Optional[int] + codingChunks=_omit, # type: Optional[int] + ): + super(ErasureCoded, self).__init__( + dataChunks=dataChunks, + codingChunks=codingChunks, + ) + + @property + def dataChunks(self): + # type: () -> int + return self._property_impl('dataChunks') + + @dataChunks.setter + def dataChunks(self, new_val): + # type: (Optional[int]) -> None + self._dataChunks = new_val + + @property + def codingChunks(self): + # type: () -> int + return self._property_impl('codingChunks') + + @codingChunks.setter + def codingChunks(self, new_val): + # type: (Optional[int]) -> None + self._codingChunks = new_val + + +class MetadataPool(CrdObject): + _properties = [ + ('failureDomain', 'failureDomain', str, False, False), + ('replicated', 'replicated', Replicated, False, False), + ('erasureCoded', 'erasureCoded', ErasureCoded, False, False) + ] + + def __init__(self, + failureDomain=_omit, # type: Optional[str] + replicated=_omit, # type: Optional[Replicated] + erasureCoded=_omit, # type: Optional[ErasureCoded] + ): + super(MetadataPool, self).__init__( + failureDomain=failureDomain, + replicated=replicated, + erasureCoded=erasureCoded, + ) + + @property + def failureDomain(self): + # type: () -> str + return self._property_impl('failureDomain') + + @failureDomain.setter + def failureDomain(self, new_val): + # type: (Optional[str]) -> None + self._failureDomain = new_val + + @property + def replicated(self): + # type: () -> Replicated + return self._property_impl('replicated') + + @replicated.setter + def replicated(self, new_val): + # type: (Optional[Replicated]) -> None + self._replicated = new_val + + @property + def erasureCoded(self): + # type: () -> ErasureCoded + return self._property_impl('erasureCoded') + + @erasureCoded.setter + def erasureCoded(self, new_val): + # type: (Optional[ErasureCoded]) -> None + self._erasureCoded = new_val + + +class DataPool(CrdObject): + _properties = [ + ('failureDomain', 'failureDomain', str, False, False), + ('replicated', 'replicated', Replicated, False, False), + ('erasureCoded', 'erasureCoded', ErasureCoded, False, False) + ] + + def __init__(self, + failureDomain=_omit, # type: Optional[str] + replicated=_omit, # type: Optional[Replicated] + erasureCoded=_omit, # type: Optional[ErasureCoded] + ): + super(DataPool, self).__init__( + failureDomain=failureDomain, + replicated=replicated, + erasureCoded=erasureCoded, + ) + + @property + def failureDomain(self): + # type: () -> str + return self._property_impl('failureDomain') + + @failureDomain.setter + def failureDomain(self, new_val): + # type: (Optional[str]) -> None + self._failureDomain = new_val + + @property + def replicated(self): + # type: () -> Replicated + return self._property_impl('replicated') + + @replicated.setter + def replicated(self, new_val): + # type: (Optional[Replicated]) -> None + self._replicated = new_val + + @property + def erasureCoded(self): + # type: () -> ErasureCoded + return self._property_impl('erasureCoded') + + @erasureCoded.setter + def erasureCoded(self, new_val): + # type: (Optional[ErasureCoded]) -> None + self._erasureCoded = new_val + + +class Spec(CrdObject): + _properties = [ + ('gateway', 'gateway', Gateway, False, False), + ('metadataPool', 'metadataPool', MetadataPool, False, False), + ('dataPool', 'dataPool', DataPool, False, False), + ('preservePoolsOnDelete', 'preservePoolsOnDelete', bool, False, False) + ] + + def __init__(self, + gateway=_omit, # type: Optional[Gateway] + metadataPool=_omit, # type: Optional[MetadataPool] + dataPool=_omit, # type: Optional[DataPool] + preservePoolsOnDelete=_omit, # type: Optional[bool] + ): + super(Spec, self).__init__( + gateway=gateway, + metadataPool=metadataPool, + dataPool=dataPool, + preservePoolsOnDelete=preservePoolsOnDelete, + ) + + @property + def gateway(self): + # type: () -> Gateway + return self._property_impl('gateway') + + @gateway.setter + def gateway(self, new_val): + # type: (Optional[Gateway]) -> None + self._gateway = new_val + + @property + def metadataPool(self): + # type: () -> MetadataPool + return self._property_impl('metadataPool') + + @metadataPool.setter + def metadataPool(self, new_val): + # type: (Optional[MetadataPool]) -> None + self._metadataPool = new_val + + @property + def dataPool(self): + # type: () -> DataPool + return self._property_impl('dataPool') + + @dataPool.setter + def dataPool(self, new_val): + # type: (Optional[DataPool]) -> None + self._dataPool = new_val + + @property + def preservePoolsOnDelete(self): + # type: () -> bool + return self._property_impl('preservePoolsOnDelete') + + @preservePoolsOnDelete.setter + def preservePoolsOnDelete(self, new_val): + # type: (Optional[bool]) -> None + self._preservePoolsOnDelete = new_val + + +class CephObjectStore(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(CephObjectStore, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/__init__.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/cluster.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/cluster.py new file mode 100644 index 0000000000000000000000000000000000000000..18501bcc2443d7fb8ecb21021187a9ad3c9095ff --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/cluster.py @@ -0,0 +1,285 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class Dashboard(CrdObject): + _properties = [ + ('localAddr', 'localAddr', str, False, False) + ] + + def __init__(self, + localAddr=_omit, # type: Optional[str] + ): + super(Dashboard, self).__init__( + localAddr=localAddr, + ) + + @property + def localAddr(self): + # type: () -> str + return self._property_impl('localAddr') + + @localAddr.setter + def localAddr(self, new_val): + # type: (Optional[str]) -> None + self._localAddr = new_val + + +class Network(CrdObject): + _properties = [ + ('serverIfName', 'serverIfName', str, False, False), + ('brokerIfName', 'brokerIfName', str, False, False) + ] + + def __init__(self, + serverIfName=_omit, # type: Optional[str] + brokerIfName=_omit, # type: Optional[str] + ): + super(Network, self).__init__( + serverIfName=serverIfName, + brokerIfName=brokerIfName, + ) + + @property + def serverIfName(self): + # type: () -> str + return self._property_impl('serverIfName') + + @serverIfName.setter + def serverIfName(self, new_val): + # type: (Optional[str]) -> None + self._serverIfName = new_val + + @property + def brokerIfName(self): + # type: () -> str + return self._property_impl('brokerIfName') + + @brokerIfName.setter + def brokerIfName(self, new_val): + # type: (Optional[str]) -> None + self._brokerIfName = new_val + + +class NodesList(CrdObjectList): + _items_type = None + + +class Storage(CrdObject): + _properties = [ + ('nodes', 'nodes', NodesList, False, False), + ('useAllDevices', 'useAllDevices', object, False, False), + ('useAllNodes', 'useAllNodes', bool, False, False) + ] + + def __init__(self, + nodes=_omit, # type: Optional[Union[List[Any], CrdObjectList]] + useAllDevices=_omit, # type: Optional[Any] + useAllNodes=_omit, # type: Optional[bool] + ): + super(Storage, self).__init__( + nodes=nodes, + useAllDevices=useAllDevices, + useAllNodes=useAllNodes, + ) + + @property + def nodes(self): + # type: () -> Union[List[Any], CrdObjectList] + return self._property_impl('nodes') + + @nodes.setter + def nodes(self, new_val): + # type: (Optional[Union[List[Any], CrdObjectList]]) -> None + self._nodes = new_val + + @property + def useAllDevices(self): + # type: () -> Any + return self._property_impl('useAllDevices') + + @useAllDevices.setter + def useAllDevices(self, new_val): + # type: (Optional[Any]) -> None + self._useAllDevices = new_val + + @property + def useAllNodes(self): + # type: () -> bool + return self._property_impl('useAllNodes') + + @useAllNodes.setter + def useAllNodes(self, new_val): + # type: (Optional[bool]) -> None + self._useAllNodes = new_val + + +class Spec(CrdObject): + _properties = [ + ('edgefsImageName', 'edgefsImageName', str, True, False), + ('dataDirHostPath', 'dataDirHostPath', str, True, False), + ('devicesResurrectMode', 'devicesResurrectMode', str, False, False), + ('dashboard', 'dashboard', Dashboard, False, False), + ('network', 'network', Network, False, False), + ('skipHostPrepare', 'skipHostPrepare', bool, False, False), + ('storage', 'storage', Storage, False, False) + ] + + def __init__(self, + edgefsImageName, # type: str + dataDirHostPath, # type: str + devicesResurrectMode=_omit, # type: Optional[str] + dashboard=_omit, # type: Optional[Dashboard] + network=_omit, # type: Optional[Network] + skipHostPrepare=_omit, # type: Optional[bool] + storage=_omit, # type: Optional[Storage] + ): + super(Spec, self).__init__( + edgefsImageName=edgefsImageName, + dataDirHostPath=dataDirHostPath, + devicesResurrectMode=devicesResurrectMode, + dashboard=dashboard, + network=network, + skipHostPrepare=skipHostPrepare, + storage=storage, + ) + + @property + def edgefsImageName(self): + # type: () -> str + return self._property_impl('edgefsImageName') + + @edgefsImageName.setter + def edgefsImageName(self, new_val): + # type: (str) -> None + self._edgefsImageName = new_val + + @property + def dataDirHostPath(self): + # type: () -> str + return self._property_impl('dataDirHostPath') + + @dataDirHostPath.setter + def dataDirHostPath(self, new_val): + # type: (str) -> None + self._dataDirHostPath = new_val + + @property + def devicesResurrectMode(self): + # type: () -> str + return self._property_impl('devicesResurrectMode') + + @devicesResurrectMode.setter + def devicesResurrectMode(self, new_val): + # type: (Optional[str]) -> None + self._devicesResurrectMode = new_val + + @property + def dashboard(self): + # type: () -> Dashboard + return self._property_impl('dashboard') + + @dashboard.setter + def dashboard(self, new_val): + # type: (Optional[Dashboard]) -> None + self._dashboard = new_val + + @property + def network(self): + # type: () -> Network + return self._property_impl('network') + + @network.setter + def network(self, new_val): + # type: (Optional[Network]) -> None + self._network = new_val + + @property + def skipHostPrepare(self): + # type: () -> bool + return self._property_impl('skipHostPrepare') + + @skipHostPrepare.setter + def skipHostPrepare(self, new_val): + # type: (Optional[bool]) -> None + self._skipHostPrepare = new_val + + @property + def storage(self): + # type: () -> Storage + return self._property_impl('storage') + + @storage.setter + def storage(self, new_val): + # type: (Optional[Storage]) -> None + self._storage = new_val + + +class Cluster(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(Cluster, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/isgw.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/isgw.py new file mode 100644 index 0000000000000000000000000000000000000000..eda8d32e497321a00b4d26ad7611e3908ffd3ae6 --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/isgw.py @@ -0,0 +1,161 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class ClientsList(CrdObjectList): + _items_type = str + + +class Config(CrdObject): + _properties = [ + ('server', 'server', str, False, False), + ('clients', 'clients', ClientsList, False, False) + ] + + def __init__(self, + server=_omit, # type: Optional[str] + clients=_omit, # type: Optional[Union[List[str], CrdObjectList]] + ): + super(Config, self).__init__( + server=server, + clients=clients, + ) + + @property + def server(self): + # type: () -> str + return self._property_impl('server') + + @server.setter + def server(self, new_val): + # type: (Optional[str]) -> None + self._server = new_val + + @property + def clients(self): + # type: () -> Union[List[str], CrdObjectList] + return self._property_impl('clients') + + @clients.setter + def clients(self, new_val): + # type: (Optional[Union[List[str], CrdObjectList]]) -> None + self._clients = new_val + + +class Spec(CrdObject): + _properties = [ + ('direction', 'direction', str, True, False), + ('remoteURL', 'remoteURL', str, False, False), + ('config', 'config', Config, False, False) + ] + + def __init__(self, + direction, # type: str + remoteURL=_omit, # type: Optional[str] + config=_omit, # type: Optional[Config] + ): + super(Spec, self).__init__( + direction=direction, + remoteURL=remoteURL, + config=config, + ) + + @property + def direction(self): + # type: () -> str + return self._property_impl('direction') + + @direction.setter + def direction(self, new_val): + # type: (str) -> None + self._direction = new_val + + @property + def remoteURL(self): + # type: () -> str + return self._property_impl('remoteURL') + + @remoteURL.setter + def remoteURL(self, new_val): + # type: (Optional[str]) -> None + self._remoteURL = new_val + + @property + def config(self): + # type: () -> Config + return self._property_impl('config') + + @config.setter + def config(self, new_val): + # type: (Optional[Config]) -> None + self._config = new_val + + +class ISGW(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(ISGW, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/nfs.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/nfs.py new file mode 100644 index 0000000000000000000000000000000000000000..ed8a9d8ed0108b17d5c0409c51eb6f20b563928c --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/nfs.py @@ -0,0 +1,95 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class Spec(CrdObject): + _properties = [ + ('instances', 'instances', int, True, False) + ] + + def __init__(self, + instances, # type: int + ): + super(Spec, self).__init__( + instances=instances, + ) + + @property + def instances(self): + # type: () -> int + return self._property_impl('instances') + + @instances.setter + def instances(self, new_val): + # type: (int) -> None + self._instances = new_val + + +class NFS(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(NFS, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3.py new file mode 100644 index 0000000000000000000000000000000000000000..a2995e1ea0235d0fe8c388f355228904e78089f3 --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3.py @@ -0,0 +1,95 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class Spec(CrdObject): + _properties = [ + ('instances', 'instances', int, True, False) + ] + + def __init__(self, + instances, # type: int + ): + super(Spec, self).__init__( + instances=instances, + ) + + @property + def instances(self): + # type: () -> int + return self._property_impl('instances') + + @instances.setter + def instances(self, new_val): + # type: (int) -> None + self._instances = new_val + + +class S3(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(S3, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3x.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3x.py new file mode 100644 index 0000000000000000000000000000000000000000..0326a8effb3d8b34e21e51bddc1f0cc8d271b45a --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3x.py @@ -0,0 +1,95 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class Spec(CrdObject): + _properties = [ + ('instances', 'instances', int, True, False) + ] + + def __init__(self, + instances, # type: int + ): + super(Spec, self).__init__( + instances=instances, + ) + + @property + def instances(self): + # type: () -> int + return self._property_impl('instances') + + @instances.setter + def instances(self, new_val): + # type: (int) -> None + self._instances = new_val + + +class S3X(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(S3X, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/swift.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/swift.py new file mode 100644 index 0000000000000000000000000000000000000000..ab4cacb589084d4f7d307af5a641f78a32cba915 --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/swift.py @@ -0,0 +1,95 @@ +""" +This file is automatically generated. +Do not modify. +""" + +try: + from typing import Any, Optional, Union, List +except ImportError: + pass + +from .._helper import _omit, CrdObject, CrdObjectList, CrdClass + +class Spec(CrdObject): + _properties = [ + ('instances', 'instances', int, True, False) + ] + + def __init__(self, + instances, # type: int + ): + super(Spec, self).__init__( + instances=instances, + ) + + @property + def instances(self): + # type: () -> int + return self._property_impl('instances') + + @instances.setter + def instances(self, new_val): + # type: (int) -> None + self._instances = new_val + + +class SWIFT(CrdClass): + _properties = [ + ('apiVersion', 'apiVersion', str, True, False), + ('metadata', 'metadata', object, True, False), + ('status', 'status', object, False, False), + ('spec', 'spec', Spec, True, False) + ] + + def __init__(self, + apiVersion, # type: str + metadata, # type: Any + spec, # type: Spec + status=_omit, # type: Optional[Any] + ): + super(SWIFT, self).__init__( + apiVersion=apiVersion, + metadata=metadata, + spec=spec, + status=status, + ) + + @property + def apiVersion(self): + # type: () -> str + return self._property_impl('apiVersion') + + @apiVersion.setter + def apiVersion(self, new_val): + # type: (str) -> None + self._apiVersion = new_val + + @property + def metadata(self): + # type: () -> Any + return self._property_impl('metadata') + + @metadata.setter + def metadata(self, new_val): + # type: (Any) -> None + self._metadata = new_val + + @property + def status(self): + # type: () -> Any + return self._property_impl('status') + + @status.setter + def status(self, new_val): + # type: (Optional[Any]) -> None + self._status = new_val + + @property + def spec(self): + # type: () -> Spec + return self._property_impl('spec') + + @spec.setter + def spec(self, new_val): + # type: (Spec) -> None + self._spec = new_val diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/tests/__init__.py b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_README.py b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_README.py new file mode 100644 index 0000000000000000000000000000000000000000..aa9261a2a55aeddc9462949c61e9988c13a8ae65 --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_README.py @@ -0,0 +1,29 @@ +def objectstore(api_name, name, namespace, instances): + from rook_client.ceph import cephobjectstore as cos + rook_os = cos.CephObjectStore( + apiVersion=api_name, + metadata=dict( + name=name, + namespace=namespace + ), + spec=cos.Spec( + metadataPool=cos.MetadataPool( + failureDomain='host', + replicated=cos.Replicated( + size=1 + ) + ), + dataPool=cos.DataPool( + failureDomain='osd', + replicated=cos.Replicated( + size=1 + ) + ), + gateway=cos.Gateway( + type='s3', + port=80, + instances=instances + ) + ) + ) + return rook_os.to_json() diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_examples.py b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_examples.py new file mode 100644 index 0000000000000000000000000000000000000000..1cfd078a5665c9d661bb94d446a06d6797b0b746 --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_examples.py @@ -0,0 +1,51 @@ +from os.path import expanduser, dirname, realpath + +import yaml +import pytest + +import rook_client +from rook_client.cassandra.cluster import Cluster as CassandraCluster +from rook_client.ceph.cephcluster import CephCluster +from rook_client.ceph.cephfilesystem import CephFilesystem +from rook_client.ceph.cephnfs import CephNFS +from rook_client.ceph.cephobjectstore import CephObjectStore +from rook_client.edgefs.cluster import Cluster as EdgefsCluster + + +def _load_example(crd_base, what): + with open(expanduser('{crd_base}/{what}').format(crd_base=crd_base, what=what)) as f: + return f.read() + + +@pytest.mark.parametrize( + "strict,cls,filename", + [ + (True, CephCluster, "ceph/cluster-external.yaml"), + (True, CephCluster, "ceph/cluster-minimal.yaml"), + (True, CephCluster, "ceph/cluster-on-pvc.yaml"), + (True, CephCluster, "ceph/cluster.yaml"), + (True, CephFilesystem, "ceph/filesystem-ec.yaml"), + (True, CephFilesystem, "ceph/filesystem-test.yaml"), + (True, CephFilesystem, "ceph/filesystem.yaml"), + (True, CephObjectStore, "ceph/object-ec.yaml"), + (True, CephObjectStore, "ceph/object-openshift.yaml"), + (True, CephObjectStore, "ceph/object-test.yaml"), + (True, CephObjectStore, "ceph/object.yaml"), + (True, CephNFS, "ceph/nfs.yaml"), + + # schema invalid: + # (False, CassandraCluster, "cassandra/cluster.yaml"), + (False, EdgefsCluster, "edgefs/cluster.yaml"), + ], +) +def test_exact_match(strict, cls, filename, crd_base): + crds = yaml.safe_load_all(_load_example(crd_base, filename)) + rook_client.STRICT = strict + [crd] = [e for e in crds if e.get('kind', '') == cls.__name__] + + c = cls.from_json(crd) + assert crd == c.to_json() + + + + diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_properties.py b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_properties.py new file mode 100644 index 0000000000000000000000000000000000000000..24ec38f5d95cee5e05b1c18b08348bd4e2b9ca23 --- /dev/null +++ b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_properties.py @@ -0,0 +1,13 @@ +from copy import deepcopy + +import pytest + +from rook_client.ceph import cephfilesystem as cfs + + +def test_omit(): + ec = cfs.ErasureCoded() + with pytest.raises(AttributeError): + ec.codingChunks + + assert not hasattr(ec, 'codingChunks') diff --git a/src/zstd/build/VS2008/zstd/zstd.vcproj b/src/zstd/build/VS2008/zstd/zstd.vcproj new file mode 100644 index 0000000000000000000000000000000000000000..745f2e875faec2fb0e09a073fd413e82d8444831 --- /dev/null +++ b/src/zstd/build/VS2008/zstd/zstd.vcproj @@ -0,0 +1,637 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/zstd/build/VS2010/zstd/zstd.rc b/src/zstd/build/VS2010/zstd/zstd.rc new file mode 100644 index 0000000000000000000000000000000000000000..f5e404730d219436b859005ed2a7b84197ac2426 --- /dev/null +++ b/src/zstd/build/VS2010/zstd/zstd.rc @@ -0,0 +1,51 @@ +// Microsoft Visual C++ generated resource script. +// + +#include "zstd.h" /* ZSTD_VERSION_STRING */ +#define APSTUDIO_READONLY_SYMBOLS +#include "verrsrc.h" +#undef APSTUDIO_READONLY_SYMBOLS + + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) +LANGUAGE 9, 1 + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// + +VS_VERSION_INFO VERSIONINFO + FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 + PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS VS_FF_DEBUG +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS_NT_WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904B0" + BEGIN + VALUE "CompanyName", "Yann Collet, Facebook, Inc." + VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm" + VALUE "FileVersion", ZSTD_VERSION_STRING + VALUE "InternalName", "zstd.exe" + VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc." + VALUE "OriginalFilename", "zstd.exe" + VALUE "ProductName", "Zstandard" + VALUE "ProductVersion", ZSTD_VERSION_STRING + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 1200 + END +END + +#endif diff --git a/src/zstd/build/VS2010/zstd/zstd.vcxproj b/src/zstd/build/VS2010/zstd/zstd.vcxproj new file mode 100644 index 0000000000000000000000000000000000000000..6e7ddca1e33d04d6f8c492be6951d9711ebe5c50 --- /dev/null +++ b/src/zstd/build/VS2010/zstd/zstd.vcxproj @@ -0,0 +1,250 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C} + Win32Proj + zstd + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + MultiByte + + + Application + true + MultiByte + + + Application + false + true + MultiByte + + + Application + false + true + MultiByte + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + false + $(LibraryPath) + + + true + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + false + $(LibraryPath); + + + false + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + false + $(LibraryPath) + + + false + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + false + $(LibraryPath); + + + + + + Level4 + Disabled + ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + setargv.obj;%(AdditionalDependencies) + + + + + + + Level4 + Disabled + ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + setargv.obj;%(AdditionalDependencies) + + + + + Level4 + + + MaxSpeed + true + true + ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + setargv.obj;%(AdditionalDependencies) + + + + + Level4 + + + MaxSpeed + true + true + ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + /DZSTD_MULTITHREAD %(AdditionalOptions) + + + Console + true + true + true + setargv.obj;%(AdditionalDependencies) + + + + + + diff --git a/src/zstd/build/cmake/lib/CMakeLists.txt b/src/zstd/build/cmake/lib/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..508bee3781983f00f45d6131498ffc13acfab1d1 --- /dev/null +++ b/src/zstd/build/cmake/lib/CMakeLists.txt @@ -0,0 +1,203 @@ +# ################################################################ +# Copyright (c) 2015-present, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + +project(libzstd) + +set(CMAKE_INCLUDE_CURRENT_DIR TRUE) +option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON) +option(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON) + +if(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC) + message(SEND_ERROR "You need to build at least one flavor of libzstd") +endif() + +# Define library directory, where sources and header files are located +include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common) + +set(Sources + ${LIBRARY_DIR}/common/entropy_common.c + ${LIBRARY_DIR}/common/fse_decompress.c + ${LIBRARY_DIR}/common/threading.c + ${LIBRARY_DIR}/common/pool.c + ${LIBRARY_DIR}/common/zstd_common.c + ${LIBRARY_DIR}/common/error_private.c + ${LIBRARY_DIR}/common/xxhash.c + ${LIBRARY_DIR}/compress/hist.c + ${LIBRARY_DIR}/compress/fse_compress.c + ${LIBRARY_DIR}/compress/huf_compress.c + ${LIBRARY_DIR}/compress/zstd_compress.c + ${LIBRARY_DIR}/compress/zstdmt_compress.c + ${LIBRARY_DIR}/compress/zstd_fast.c + ${LIBRARY_DIR}/compress/zstd_double_fast.c + ${LIBRARY_DIR}/compress/zstd_lazy.c + ${LIBRARY_DIR}/compress/zstd_opt.c + ${LIBRARY_DIR}/compress/zstd_ldm.c + ${LIBRARY_DIR}/decompress/huf_decompress.c + ${LIBRARY_DIR}/decompress/zstd_decompress.c + ${LIBRARY_DIR}/decompress/zstd_decompress_block.c + ${LIBRARY_DIR}/decompress/zstd_ddict.c + ${LIBRARY_DIR}/dictBuilder/cover.c + ${LIBRARY_DIR}/dictBuilder/fastcover.c + ${LIBRARY_DIR}/dictBuilder/divsufsort.c + ${LIBRARY_DIR}/dictBuilder/zdict.c + ${LIBRARY_DIR}/deprecated/zbuff_common.c + ${LIBRARY_DIR}/deprecated/zbuff_compress.c + ${LIBRARY_DIR}/deprecated/zbuff_decompress.c) + +set(Headers + ${LIBRARY_DIR}/zstd.h + ${LIBRARY_DIR}/common/debug.h + ${LIBRARY_DIR}/common/pool.h + ${LIBRARY_DIR}/common/threading.h + ${LIBRARY_DIR}/common/bitstream.h + ${LIBRARY_DIR}/common/error_private.h + ${LIBRARY_DIR}/common/zstd_errors.h + ${LIBRARY_DIR}/common/fse.h + ${LIBRARY_DIR}/common/huf.h + ${LIBRARY_DIR}/common/mem.h + ${LIBRARY_DIR}/common/zstd_internal.h + ${LIBRARY_DIR}/compress/hist.h + ${LIBRARY_DIR}/compress/zstd_compress_internal.h + ${LIBRARY_DIR}/compress/zstd_fast.h + ${LIBRARY_DIR}/compress/zstd_double_fast.h + ${LIBRARY_DIR}/compress/zstd_lazy.h + ${LIBRARY_DIR}/compress/zstd_opt.h + ${LIBRARY_DIR}/compress/zstd_ldm.h + ${LIBRARY_DIR}/compress/zstdmt_compress.h + ${LIBRARY_DIR}/decompress/zstd_decompress_internal.h + ${LIBRARY_DIR}/decompress/zstd_decompress_block.h + ${LIBRARY_DIR}/decompress/zstd_ddict.h + ${LIBRARY_DIR}/dictBuilder/zdict.h + ${LIBRARY_DIR}/dictBuilder/cover.h + ${LIBRARY_DIR}/deprecated/zbuff.h) + +if (ZSTD_LEGACY_SUPPORT) + set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy) + include_directories(${LIBRARY_LEGACY_DIR}) + + set(Sources ${Sources} + ${LIBRARY_LEGACY_DIR}/zstd_v01.c + ${LIBRARY_LEGACY_DIR}/zstd_v02.c + ${LIBRARY_LEGACY_DIR}/zstd_v03.c + ${LIBRARY_LEGACY_DIR}/zstd_v04.c + ${LIBRARY_LEGACY_DIR}/zstd_v05.c + ${LIBRARY_LEGACY_DIR}/zstd_v06.c + ${LIBRARY_LEGACY_DIR}/zstd_v07.c) + + set(Headers ${Headers} + ${LIBRARY_LEGACY_DIR}/zstd_legacy.h + ${LIBRARY_LEGACY_DIR}/zstd_v01.h + ${LIBRARY_LEGACY_DIR}/zstd_v02.h + ${LIBRARY_LEGACY_DIR}/zstd_v03.h + ${LIBRARY_LEGACY_DIR}/zstd_v04.h + ${LIBRARY_LEGACY_DIR}/zstd_v05.h + ${LIBRARY_LEGACY_DIR}/zstd_v06.h + ${LIBRARY_LEGACY_DIR}/zstd_v07.h) +endif () + +if (MSVC) + set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll) + set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc) +endif () + +# Split project to static and shared libraries build +if (ZSTD_BUILD_SHARED) + add_library(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources}) + if (ZSTD_MULTITHREAD_SUPPORT) + set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD") + if (UNIX) + target_link_libraries(libzstd_shared ${THREADS_LIBS}) + endif () + endif() +endif () +if (ZSTD_BUILD_STATIC) + add_library(libzstd_static STATIC ${Sources} ${Headers}) + if (ZSTD_MULTITHREAD_SUPPORT) + set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD") + if (UNIX) + target_link_libraries(libzstd_static ${THREADS_LIBS}) + endif () + endif () +endif () + +# Add specific compile definitions for MSVC project +if (MSVC) + if (ZSTD_BUILD_SHARED) + set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS") + endif () + if (ZSTD_BUILD_STATIC) + set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS") + endif () +endif () + +# With MSVC static library needs to be renamed to avoid conflict with import library +if (MSVC) + set(STATIC_LIBRARY_BASE_NAME zstd_static) +else () + set(STATIC_LIBRARY_BASE_NAME zstd) +endif () + +# Define static and shared library names +if (ZSTD_BUILD_SHARED) + set_target_properties( + libzstd_shared + PROPERTIES + OUTPUT_NAME zstd + VERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH} + SOVERSION ${zstd_VERSION_MAJOR}) +endif () + +if (ZSTD_BUILD_STATIC) + set_target_properties( + libzstd_static + PROPERTIES + OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME}) +endif () + +if (UNIX) + # pkg-config + set(PREFIX "${CMAKE_INSTALL_PREFIX}") + set(LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") + set(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include") + set(VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}") + add_custom_target(libzstd.pc ALL + ${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc" + -DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake" + COMMENT "Creating pkg-config file") + + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${LIBDIR}/pkgconfig") +endif () + +# install target +install(FILES + ${LIBRARY_DIR}/zstd.h + ${LIBRARY_DIR}/deprecated/zbuff.h + ${LIBRARY_DIR}/dictBuilder/zdict.h + ${LIBRARY_DIR}/dictBuilder/cover.h + ${LIBRARY_DIR}/common/zstd_errors.h + DESTINATION "include") + +if (ZSTD_BUILD_SHARED) + install(TARGETS libzstd_shared RUNTIME DESTINATION "bin" + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") +endif() +if (ZSTD_BUILD_STATIC) + install(TARGETS libzstd_static ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") +endif () + +# uninstall target +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" + IMMEDIATE @ONLY) + +add_custom_target(uninstall + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) diff --git a/src/zstd/build/cmake/lib/cmake_uninstall.cmake.in b/src/zstd/build/cmake/lib/cmake_uninstall.cmake.in new file mode 100644 index 0000000000000000000000000000000000000000..9f1d045ddfab1e1c004f85c319e7080255fa6a25 --- /dev/null +++ b/src/zstd/build/cmake/lib/cmake_uninstall.cmake.in @@ -0,0 +1,22 @@ + +if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt") + message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt") +endif() + +file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files) +string(REGEX REPLACE "\n" ";" files "${files}") +foreach(file ${files}) + message(STATUS "Uninstalling $ENV{DESTDIR}${file}") + if(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") + exec_program( + "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" + OUTPUT_VARIABLE rm_out + RETURN_VALUE rm_retval + ) + if(NOT "${rm_retval}" STREQUAL 0) + message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}") + endif() + else() + message(STATUS "File $ENV{DESTDIR}${file} does not exist.") + endif() +endforeach() diff --git a/src/zstd/build/cmake/lib/pkgconfig.cmake b/src/zstd/build/cmake/lib/pkgconfig.cmake new file mode 100644 index 0000000000000000000000000000000000000000..8f805a1970348c116a9c4b980875dc15cdad1f04 --- /dev/null +++ b/src/zstd/build/cmake/lib/pkgconfig.cmake @@ -0,0 +1 @@ +configure_file("${IN}" "${OUT}" @ONLY) diff --git a/src/zstd/build/meson/lib/meson.build b/src/zstd/build/meson/lib/meson.build new file mode 100644 index 0000000000000000000000000000000000000000..f8014c625863ff877855505792b04b375a4ae821 --- /dev/null +++ b/src/zstd/build/meson/lib/meson.build @@ -0,0 +1,129 @@ +# ############################################################################# +# Copyright (c) 2018-present Dima Krasner +# lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# + +zstd_rootdir = '../../..' + +libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'), + join_paths(zstd_rootdir, 'lib/common'), + join_paths(zstd_rootdir, 'lib/compress'), + join_paths(zstd_rootdir, 'lib/decompress'), + join_paths(zstd_rootdir, 'lib/dictBuilder'), + join_paths(zstd_rootdir, 'lib/deprecated'))] + +libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'), + join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'), + join_paths(zstd_rootdir, 'lib/common/threading.c'), + join_paths(zstd_rootdir, 'lib/common/pool.c'), + join_paths(zstd_rootdir, 'lib/common/zstd_common.c'), + join_paths(zstd_rootdir, 'lib/common/error_private.c'), + join_paths(zstd_rootdir, 'lib/common/xxhash.c'), + join_paths(zstd_rootdir, 'lib/compress/hist.c'), + join_paths(zstd_rootdir, 'lib/compress/fse_compress.c'), + join_paths(zstd_rootdir, 'lib/compress/huf_compress.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'), + join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_lazy.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_opt.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_ldm.c'), + join_paths(zstd_rootdir, 'lib/decompress/huf_decompress.c'), + join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress.c'), + join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress_block.c'), + join_paths(zstd_rootdir, 'lib/decompress/zstd_ddict.c'), + join_paths(zstd_rootdir, 'lib/dictBuilder/cover.c'), + join_paths(zstd_rootdir, 'lib/dictBuilder/fastcover.c'), + join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'), + join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c'), + join_paths(zstd_rootdir, 'lib/deprecated/zbuff_common.c'), + join_paths(zstd_rootdir, 'lib/deprecated/zbuff_compress.c'), + join_paths(zstd_rootdir, 'lib/deprecated/zbuff_decompress.c')] + +# Explicit define legacy support +add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_level), + language: 'c') + +if legacy_level == 0 + message('Legacy support: DISABLED') +else + # See ZSTD_LEGACY_SUPPORT of lib/README.md + message('Enable legacy support back to version 0.@0@'.format(legacy_level)) + + libzstd_includes += [ include_directories(join_paths(zstd_rootdir, 'lib/legacy')) ] + foreach i : [1, 2, 3, 4, 5, 6, 7] + if legacy_level <= i + libzstd_sources += join_paths(zstd_rootdir, 'lib/legacy/zstd_v0@0@.c'.format(i)) + endif + endforeach +endif + +libzstd_deps = [] +if use_multi_thread + message('Enable multi-threading support') + add_project_arguments('-DZSTD_MULTITHREAD', language: 'c') + libzstd_deps = [ thread_dep ] +endif + +libzstd_c_args = [] +if cc_id == compiler_msvc + if default_library_type != 'static' + libzstd_sources += [windows_mod.compile_resources( + join_paths(zstd_rootdir, 'build/VS2010/libzstd-dll/libzstd-dll.rc'))] + libzstd_c_args += ['-DZSTD_DLL_EXPORT=1', + '-DZSTD_HEAPMODE=0', + '-D_CONSOLE', + '-D_CRT_SECURE_NO_WARNINGS'] + else + libzstd_c_args += ['-DZSTD_HEAPMODE=0', + '-D_CRT_SECURE_NO_WARNINGS'] + endif +endif + +mingw_ansi_stdio_flags = [] +if host_machine_os == os_windows and cc_id == compiler_gcc + mingw_ansi_stdio_flags = [ '-D__USE_MINGW_ANSI_STDIO' ] +endif +libzstd_c_args += mingw_ansi_stdio_flags + +libzstd_debug_cflags = [] +if use_debug + libzstd_c_args += '-DDEBUGLEVEL=@0@'.format(debug_level) + if cc_id == compiler_gcc or cc_id == compiler_clang + libzstd_debug_cflags = ['-Wstrict-aliasing=1', '-Wswitch-enum', + '-Wdeclaration-after-statement', '-Wstrict-prototypes', + '-Wundef', '-Wpointer-arith', '-Wvla', + '-Wformat=2', '-Winit-self', '-Wfloat-equal', '-Wwrite-strings', + '-Wredundant-decls', '-Wmissing-prototypes', '-Wc++-compat'] + endif +endif +libzstd_c_args += cc.get_supported_arguments(libzstd_debug_cflags) + +libzstd = library('zstd', + libzstd_sources, + include_directories: libzstd_includes, + c_args: libzstd_c_args, + dependencies: libzstd_deps, + install: true, + version: zstd_libversion) + +libzstd_dep = declare_dependency(link_with: libzstd, + include_directories: libzstd_includes) + +pkgconfig.generate(libzstd, + name: 'libzstd', + filebase: 'libzstd', + description: 'fast lossless compression algorithm library', + version: zstd_libversion, + url: 'http://www.zstd.net/') + +install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'), + join_paths(zstd_rootdir, 'lib/deprecated/zbuff.h'), + join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.h'), + join_paths(zstd_rootdir, 'lib/common/zstd_errors.h')) diff --git a/src/zstd/contrib/VS2005/zstd/zstd.vcproj b/src/zstd/contrib/VS2005/zstd/zstd.vcproj new file mode 100644 index 0000000000000000000000000000000000000000..46cabbf6e3ddef7938ea748471f6bfaf91ca0cdb --- /dev/null +++ b/src/zstd/contrib/VS2005/zstd/zstd.vcproj @@ -0,0 +1,548 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/zstd/examples/dictionary_compression.c b/src/zstd/examples/dictionary_compression.c new file mode 100644 index 0000000000000000000000000000000000000000..9efdb785c1127a610484b67b730a90d6de7298e2 --- /dev/null +++ b/src/zstd/examples/dictionary_compression.c @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#include // printf +#include // free +#include // memset, strcat +#include // presumes zstd library is installed +#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() + +/* createDict() : + `dictFileName` is supposed to have been created using `zstd --train` */ +static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel) +{ + size_t dictSize; + printf("loading dictionary %s \n", dictFileName); + void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize); + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel); + CHECK(cdict != NULL, "ZSTD_createCDict() failed!"); + free(dictBuffer); + return cdict; +} + + +static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict) +{ + size_t fSize; + void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize); + size_t const cBuffSize = ZSTD_compressBound(fSize); + void* const cBuff = malloc_orDie(cBuffSize); + + /* Compress using the dictionary. + * This function writes the dictionary id, and content size into the header. + * But, it doesn't use a checksum. You can control these options using the + * advanced API: ZSTD_CCtx_setParameter(), ZSTD_CCtx_refCDict(), + * and ZSTD_compress2(). + */ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + CHECK(cctx != NULL, "ZSTD_createCCtx() failed!"); + size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict); + CHECK_ZSTD(cSize); + + saveFile_orDie(oname, cBuff, cSize); + + /* success */ + printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname); + + ZSTD_freeCCtx(cctx); /* never fails */ + free(fBuff); + free(cBuff); +} + + +static char* createOutFilename_orDie(const char* filename) +{ + size_t const inL = strlen(filename); + size_t const outL = inL + 5; + void* outSpace = malloc_orDie(outL); + memset(outSpace, 0, outL); + strcat(outSpace, filename); + strcat(outSpace, ".zst"); + return (char*)outSpace; +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + int const cLevel = 3; + + if (argc<3) { + fprintf(stderr, "wrong arguments\n"); + fprintf(stderr, "usage:\n"); + fprintf(stderr, "%s [FILES] dictionary\n", exeName); + return 1; + } + + /* load dictionary only once */ + const char* const dictName = argv[argc-1]; + ZSTD_CDict* const dictPtr = createCDict_orDie(dictName, cLevel); + + int u; + for (u=1; u // printf +#include // free +#include // presumes zstd library is installed +#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() + +/* createDict() : + `dictFileName` is supposed to have been created using `zstd --train` */ +static ZSTD_DDict* createDict_orDie(const char* dictFileName) +{ + size_t dictSize; + printf("loading dictionary %s \n", dictFileName); + void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize); + ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize); + CHECK(ddict != NULL, "ZSTD_createDDict() failed!"); + free(dictBuffer); + return ddict; +} + +static void decompress(const char* fname, const ZSTD_DDict* ddict) +{ + size_t cSize; + void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize); + /* Read the content size from the frame header. For simplicity we require + * that it is always present. By default, zstd will write the content size + * in the header when it is known. If you can't guarantee that the frame + * content size is always written into the header, either use streaming + * decompression, or ZSTD_decompressBound(). + */ + unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize); + CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname); + CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname); + void* const rBuff = malloc_orDie((size_t)rSize); + + /* Check that the dictionary ID matches. + * If a non-zstd dictionary is used, then both will be zero. + * By default zstd always writes the dictionary ID into the frame. + * Zstd will check if there is a dictionary ID mismatch as well. + */ + unsigned const expectedDictID = ZSTD_getDictID_fromDDict(ddict); + unsigned const actualDictID = ZSTD_getDictID_fromFrame(cBuff, cSize); + CHECK(actualDictID == expectedDictID, + "DictID mismatch: expected %u got %u", + expectedDictID, + actualDictID); + + /* Decompress using the dictionary. + * If you need to control the decompression parameters, then use the + * advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and + * ZSTD_decompressDCtx(). + */ + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + CHECK(dctx != NULL, "ZSTD_createDCtx() failed!"); + size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict); + CHECK_ZSTD(dSize); + /* When zstd knows the content size, it will error if it doesn't match. */ + CHECK(dSize == rSize, "Impossible because zstd will check this condition!"); + + /* success */ + printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize); + + ZSTD_freeDCtx(dctx); + free(rBuff); + free(cBuff); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc<3) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s [FILES] dictionary\n", exeName); + return 1; + } + + /* load dictionary only once */ + const char* const dictName = argv[argc-1]; + ZSTD_DDict* const dictPtr = createDict_orDie(dictName); + + int u; + for (u=1; u +#include +#include +#include "fuzz_helpers.h" +#include "zstd_helpers.h" + +static ZSTD_DCtx *dctx = NULL; +static void* rBuf = NULL; +static size_t bufSize = 0; + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + FUZZ_dict_t dict; + size_t neededBufSize; + + uint32_t seed = FUZZ_seed(&src, &size); + neededBufSize = MAX(20 * size, (size_t)256 << 10); + + /* Allocate all buffers and contexts if not already allocated */ + if (neededBufSize > bufSize) { + free(rBuf); + rBuf = malloc(neededBufSize); + bufSize = neededBufSize; + FUZZ_ASSERT(rBuf); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + dict = FUZZ_train(src, size, &seed); + if (FUZZ_rand32(&seed, 0, 1) == 0) { + ZSTD_decompress_usingDict(dctx, + rBuf, neededBufSize, + src, size, + dict.buff, dict.size); + } else { + FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( + dctx, dict.buff, dict.size, + (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + (ZSTD_dictContentType_e)FUZZ_rand32(&seed, 0, 2))); + ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, src, size); + } + + free(dict.buff); +#ifndef STATEFUL_FUZZING + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/fuzz/dictionary_round_trip.c b/src/zstd/tests/fuzz/dictionary_round_trip.c new file mode 100644 index 0000000000000000000000000000000000000000..e28c65c98f06d94d8889bd8e1a4e1ba3c9c97c29 --- /dev/null +++ b/src/zstd/tests/fuzz/dictionary_round_trip.c @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +/** + * This fuzz target performs a zstd round-trip test (compress & decompress) with + * a dictionary, compares the result with the original, and calls abort() on + * corruption. + */ + +#include +#include +#include +#include +#include "fuzz_helpers.h" +#include "zstd_helpers.h" + +static const int kMaxClevel = 19; + +static ZSTD_CCtx *cctx = NULL; +static ZSTD_DCtx *dctx = NULL; +static uint32_t seed; + +static size_t roundTripTest(void *result, size_t resultCapacity, + void *compressed, size_t compressedCapacity, + const void *src, size_t srcSize) +{ + ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto; + FUZZ_dict_t dict = FUZZ_train(src, srcSize, &seed); + size_t cSize; + if ((FUZZ_rand(&seed) & 15) == 0) { + int const cLevel = FUZZ_rand(&seed) % kMaxClevel; + + cSize = ZSTD_compress_usingDict(cctx, + compressed, compressedCapacity, + src, srcSize, + dict.buff, dict.size, + cLevel); + } else { + dictContentType = FUZZ_rand32(&seed, 0, 2); + FUZZ_setRandomParameters(cctx, srcSize, &seed); + /* Disable checksum so we can use sizes smaller than compress bound. */ + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)); + FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced( + cctx, dict.buff, dict.size, + (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + dictContentType)); + cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); + } + FUZZ_ZASSERT(cSize); + FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( + dctx, dict.buff, dict.size, + (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + dictContentType)); + { + size_t const ret = ZSTD_decompressDCtx( + dctx, result, resultCapacity, compressed, cSize); + free(dict.buff); + return ret; + } +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + size_t const rBufSize = size; + void* rBuf = malloc(rBufSize); + size_t cBufSize = ZSTD_compressBound(size); + void* cBuf; + + seed = FUZZ_seed(&src, &size); + /* Half of the time fuzz with a 1 byte smaller output size. + * This will still succeed because we force the checksum to be disabled, + * giving us 4 bytes of overhead. + */ + cBufSize -= FUZZ_rand32(&seed, 0, 1); + cBuf = malloc(cBufSize); + + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + { + size_t const result = + roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size); + FUZZ_ZASSERT(result); + FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); + } + free(rBuf); + free(cBuf); +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/gzip/hufts-segv.gz b/src/zstd/tests/gzip/hufts-segv.gz new file mode 100644 index 0000000000000000000000000000000000000000..32cb2a256844358eca0b5e78e49b96a60724ade5 Binary files /dev/null and b/src/zstd/tests/gzip/hufts-segv.gz differ diff --git a/src/zstd/zlibWrapper/examples/example.c b/src/zstd/zlibWrapper/examples/example.c new file mode 100644 index 0000000000000000000000000000000000000000..9000f7a3295ecd5b4b44db4ed843a036bda8a421 --- /dev/null +++ b/src/zstd/zlibWrapper/examples/example.c @@ -0,0 +1,629 @@ +/* example.c contains minimal changes required to be compiled with zlibWrapper: + * - #include "zlib.h" was changed to #include "zstd_zlibwrapper.h" + * - test_flush() and test_sync() use functions not supported by zlibWrapper + therefore they are disabled while zstd compression is turned on */ + +/* example.c -- usage example of the zlib compression library + */ +/* + Copyright (c) 1995-2006, 2011 Jean-loup Gailly + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgement in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + */ + +/* @(#) $Id$ */ + +#include "zstd_zlibwrapper.h" +#include + +#ifdef STDC +# include +# include +#endif + +#if defined(VMS) || defined(RISCOS) +# define TESTFILE "foo-gz" +#else +# define TESTFILE "foo.gz" +#endif + +#define CHECK_ERR(err, msg) { \ + if (err != Z_OK) { \ + fprintf(stderr, "%s error: %d\n", msg, err); \ + exit(1); \ + } \ +} + +z_const char hello[] = "hello, hello! I said hello, hello!"; +/* "hello world" would be more standard, but the repeated "hello" + * stresses the compression code better, sorry... + */ + +const char dictionary[] = "hello, hello!"; +uLong dictId; /* Adler32 value of the dictionary */ + +void test_deflate OF((Byte *compr, uLong comprLen)); +void test_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_large_deflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_large_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_flush OF((Byte *compr, uLong *comprLen)); +void test_sync OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_dict_deflate OF((Byte *compr, uLong comprLen)); +void test_dict_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +int main OF((int argc, char *argv[])); + + +#ifdef Z_SOLO + +void *myalloc OF((void *, unsigned, unsigned)); +void myfree OF((void *, void *)); + +void *myalloc(q, n, m) + void *q; + unsigned n, m; +{ + void *buf = calloc(n, m); + q = Z_NULL; + /* printf("myalloc %p n=%d m=%d\n", buf, n, m); */ + return buf; +} + +void myfree(void *q, void *p) +{ + /* printf("myfree %p\n", p); */ + q = Z_NULL; + free(p); +} + +static alloc_func zalloc = myalloc; +static free_func zfree = myfree; + +#else /* !Z_SOLO */ + +static alloc_func zalloc = (alloc_func)0; +static free_func zfree = (free_func)0; + +void test_compress OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_gzio OF((const char *fname, + Byte *uncompr, uLong uncomprLen)); + +/* =========================================================================== + * Test compress() and uncompress() + */ +void test_compress(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + uLong len = (uLong)strlen(hello)+1; + + err = compress(compr, &comprLen, (const Bytef*)hello, len); + CHECK_ERR(err, "compress"); + + strcpy((char*)uncompr, "garbage"); + + err = uncompress(uncompr, &uncomprLen, compr, comprLen); + CHECK_ERR(err, "uncompress"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad uncompress\n"); + exit(1); + } else { + printf("uncompress(): %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Test read/write of .gz files + */ +void test_gzio(fname, uncompr, uncomprLen) + const char *fname; /* compressed file name */ + Byte *uncompr; + uLong uncomprLen; +{ +#ifdef NO_GZCOMPRESS + fprintf(stderr, "NO_GZCOMPRESS -- gz* functions cannot compress\n"); +#else + int err; + int len = (int)strlen(hello)+1; + gzFile file; + z_off_t pos; + + file = gzopen(fname, "wb"); + if (file == NULL) { + fprintf(stderr, "gzopen error\n"); + exit(1); + } + gzputc(file, 'h'); + if (gzputs(file, "ello") != 4) { + fprintf(stderr, "gzputs err: %s\n", gzerror(file, &err)); + exit(1); + } + if (gzprintf(file, ", %s! I said hello, hello!", "hello") != 8+21) { + fprintf(stderr, "gzprintf err: %s\n", gzerror(file, &err)); + exit(1); + } + gzseek(file, 1L, SEEK_CUR); /* add one zero byte */ + gzclose(file); + + file = gzopen(fname, "rb"); + if (file == NULL) { + fprintf(stderr, "gzopen error\n"); + exit(1); + } + strcpy((char*)uncompr, "garbage"); + + if (gzread(file, uncompr, (unsigned)uncomprLen) != len) { + fprintf(stderr, "gzread err: %s\n", gzerror(file, &err)); + exit(1); + } + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad gzread: %s\n", (char*)uncompr); + exit(1); + } else { + printf("gzread(): %s\n", (char*)uncompr); + } + + pos = gzseek(file, -8L, SEEK_CUR); + if (pos != 6+21 || gztell(file) != pos) { + fprintf(stderr, "gzseek error, pos=%ld, gztell=%ld\n", + (long)pos, (long)gztell(file)); + exit(1); + } + + if (gzgetc(file) != ' ') { + fprintf(stderr, "gzgetc error\n"); + exit(1); + } + + if (gzungetc(' ', file) != ' ') { + fprintf(stderr, "gzungetc error\n"); + exit(1); + } + + gzgets(file, (char*)uncompr, (int)uncomprLen); + if (strlen((char*)uncompr) != 7) { /* " hello!" */ + fprintf(stderr, "gzgets err after gzseek: %s\n", gzerror(file, &err)); + exit(1); + } + if (strcmp((char*)uncompr, hello + 6+21)) { + fprintf(stderr, "bad gzgets after gzseek\n"); + exit(1); + } else { + printf("gzgets() after gzseek: %s\n", (char*)uncompr); + } + + gzclose(file); +#endif +} + +#endif /* Z_SOLO */ + +/* =========================================================================== + * Test deflate() with small buffers + */ +void test_deflate(compr, comprLen) + Byte *compr; + uLong comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + uLong len = (uLong)strlen(hello)+1; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.next_out = compr; + + while (c_stream.total_in != len && c_stream.total_out < comprLen) { + c_stream.avail_in = c_stream.avail_out = 1; /* force small buffers */ + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + } + /* Finish the stream, still forcing small buffers: */ + for (;;) { + c_stream.avail_out = 1; + err = deflate(&c_stream, Z_FINISH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "deflate"); + } + + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with small buffers + */ +void test_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = 0; + d_stream.next_out = uncompr; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + while (d_stream.total_out < uncomprLen && d_stream.total_in < comprLen) { + d_stream.avail_in = d_stream.avail_out = 1; /* force small buffers */ + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "inflate"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad inflate\n"); + exit(1); + } else { + printf("inflate(): %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Test deflate() with large buffers and dynamic change of compression level + */ +void test_large_deflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_BEST_SPEED); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_out = compr; + c_stream.avail_out = (uInt)comprLen; + + /* At this point, uncompr is still mostly zeroes, so it should compress + * very well: + */ + c_stream.next_in = uncompr; + c_stream.avail_in = (uInt)uncomprLen; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + if (c_stream.avail_in != 0) { + fprintf(stderr, "deflate not greedy\n"); + exit(1); + } + + /* Feed in already compressed data and switch to no compression: */ + deflateParams(&c_stream, Z_NO_COMPRESSION, Z_DEFAULT_STRATEGY); + c_stream.next_in = compr; + c_stream.avail_in = (uInt)comprLen/2; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + + /* Switch back to compressing mode: */ + deflateParams(&c_stream, Z_BEST_COMPRESSION, Z_FILTERED); + c_stream.next_in = uncompr; + c_stream.avail_in = (uInt)uncomprLen; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + fprintf(stderr, "deflate should report Z_STREAM_END\n"); + exit(1); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with large buffers + */ +void test_large_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = (uInt)comprLen; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + for (;;) { + d_stream.next_out = uncompr; /* discard the output */ + d_stream.avail_out = (uInt)uncomprLen; + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "large inflate"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (d_stream.total_out != 2*uncomprLen + comprLen/2) { + fprintf(stderr, "bad large inflate: %ld\n", d_stream.total_out); + exit(1); + } else { + printf("large_inflate(): OK\n"); + } +} + +/* =========================================================================== + * Test deflate() with full flush + */ +void test_flush(compr, comprLen) + Byte *compr; + uLong *comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + uInt len = (uInt)strlen(hello)+1; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.next_out = compr; + c_stream.avail_in = 3; + c_stream.avail_out = (uInt)*comprLen; + err = deflate(&c_stream, Z_FULL_FLUSH); + CHECK_ERR(err, "deflate"); + + compr[3]++; /* force an error in first compressed block */ + c_stream.avail_in = len - 3; + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + CHECK_ERR(err, "deflate"); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); + + *comprLen = c_stream.total_out; +} + +/* =========================================================================== + * Test inflateSync() + */ +void test_sync(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = 2; /* just read the zlib header */ + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + d_stream.next_out = uncompr; + d_stream.avail_out = (uInt)uncomprLen; + + inflate(&d_stream, Z_NO_FLUSH); + CHECK_ERR(err, "inflate"); + + d_stream.avail_in = (uInt)comprLen-2; /* read all compressed data */ + err = inflateSync(&d_stream); /* but skip the damaged part */ + CHECK_ERR(err, "inflateSync"); + + err = inflate(&d_stream, Z_FINISH); + if (err != Z_DATA_ERROR) { + fprintf(stderr, "inflate should report DATA_ERROR\n"); + /* Because of incorrect adler32 */ + exit(1); + } + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + printf("after inflateSync(): hel%s\n", (char *)uncompr); +} + +/* =========================================================================== + * Test deflate() with preset dictionary + */ +void test_dict_deflate(compr, comprLen) + Byte *compr; + uLong comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_BEST_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + err = deflateSetDictionary(&c_stream, + (const Bytef*)dictionary, (int)sizeof(dictionary)); + CHECK_ERR(err, "deflateSetDictionary"); + + dictId = c_stream.adler; + c_stream.next_out = compr; + c_stream.avail_out = (uInt)comprLen; + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.avail_in = (uInt)strlen(hello)+1; + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + fprintf(stderr, "deflate should report Z_STREAM_END\n"); + exit(1); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with a preset dictionary + */ +void test_dict_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = (uInt)comprLen; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + d_stream.next_out = uncompr; + d_stream.avail_out = (uInt)uncomprLen; + + for (;;) { + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + if (err == Z_NEED_DICT) { + if (d_stream.adler != dictId) { + fprintf(stderr, "unexpected dictionary"); + exit(1); + } + err = inflateSetDictionary(&d_stream, (const Bytef*)dictionary, + (int)sizeof(dictionary)); + } + CHECK_ERR(err, "inflate with dict"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad inflate with dict\n"); + exit(1); + } else { + printf("inflate with dictionary: %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Usage: example [output.gz [input.gz]] + */ + +int main(argc, argv) + int argc; + char *argv[]; +{ + Byte *compr, *uncompr; + uLong comprLen = 10000*sizeof(int); /* don't overflow on MSDOS */ + uLong uncomprLen = comprLen; + static const char* myVersion = ZLIB_VERSION; + + if (zlibVersion()[0] != myVersion[0]) { + fprintf(stderr, "incompatible zlib version\n"); + exit(1); + + } else if (strcmp(zlibVersion(), ZLIB_VERSION) != 0) { + fprintf(stderr, "warning: different zlib version\n"); + } + + printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n", + ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags()); + if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); + + compr = (Byte*)calloc((uInt)comprLen, 1); + uncompr = (Byte*)calloc((uInt)uncomprLen, 1); + /* compr and uncompr are cleared to avoid reading uninitialized + * data and to ensure that uncompr compresses well. + */ + if (compr == Z_NULL || uncompr == Z_NULL) { + printf("out of memory\n"); + exit(1); + } + +#ifdef Z_SOLO + argc = strlen(argv[0]); +#else + test_compress(compr, comprLen, uncompr, uncomprLen); + + test_gzio((argc > 1 ? argv[1] : TESTFILE), + uncompr, uncomprLen); +#endif + + test_deflate(compr, comprLen); + test_inflate(compr, comprLen, uncompr, uncomprLen); + + test_large_deflate(compr, comprLen, uncompr, uncomprLen); + test_large_inflate(compr, comprLen, uncompr, uncomprLen); + + if (!ZWRAP_isUsingZSTDcompression()) { + test_flush(compr, &comprLen); + test_sync(compr, comprLen, uncompr, uncomprLen); + } + comprLen = uncomprLen; + + test_dict_deflate(compr, comprLen); + test_dict_inflate(compr, comprLen, uncompr, uncomprLen); + + free(compr); + free(uncompr); + + return 0; +} diff --git a/src/zstd/zlibWrapper/examples/fitblk.c b/src/zstd/zlibWrapper/examples/fitblk.c new file mode 100644 index 0000000000000000000000000000000000000000..6418ca38763c3a2c94c6e689210a818fec19ad8a --- /dev/null +++ b/src/zstd/zlibWrapper/examples/fitblk.c @@ -0,0 +1,254 @@ +/* fitblk.c contains minimal changes required to be compiled with zlibWrapper: + * - #include "zlib.h" was changed to #include "zstd_zlibwrapper.h" + * - writing block to stdout was disabled */ + +/* fitblk.c: example of fitting compressed output to a specified size + Not copyrighted -- provided to the public domain + Version 1.1 25 November 2004 Mark Adler */ + +/* Version history: + 1.0 24 Nov 2004 First version + 1.1 25 Nov 2004 Change deflateInit2() to deflateInit() + Use fixed-size, stack-allocated raw buffers + Simplify code moving compression to subroutines + Use assert() for internal errors + Add detailed description of approach + */ + +/* Approach to just fitting a requested compressed size: + + fitblk performs three compression passes on a portion of the input + data in order to determine how much of that input will compress to + nearly the requested output block size. The first pass generates + enough deflate blocks to produce output to fill the requested + output size plus a specified excess amount (see the EXCESS define + below). The last deflate block may go quite a bit past that, but + is discarded. The second pass decompresses and recompresses just + the compressed data that fit in the requested plus excess sized + buffer. The deflate process is terminated after that amount of + input, which is less than the amount consumed on the first pass. + The last deflate block of the result will be of a comparable size + to the final product, so that the header for that deflate block and + the compression ratio for that block will be about the same as in + the final product. The third compression pass decompresses the + result of the second step, but only the compressed data up to the + requested size minus an amount to allow the compressed stream to + complete (see the MARGIN define below). That will result in a + final compressed stream whose length is less than or equal to the + requested size. Assuming sufficient input and a requested size + greater than a few hundred bytes, the shortfall will typically be + less than ten bytes. + + If the input is short enough that the first compression completes + before filling the requested output size, then that compressed + stream is return with no recompression. + + EXCESS is chosen to be just greater than the shortfall seen in a + two pass approach similar to the above. That shortfall is due to + the last deflate block compressing more efficiently with a smaller + header on the second pass. EXCESS is set to be large enough so + that there is enough uncompressed data for the second pass to fill + out the requested size, and small enough so that the final deflate + block of the second pass will be close in size to the final deflate + block of the third and final pass. MARGIN is chosen to be just + large enough to assure that the final compression has enough room + to complete in all cases. + */ + +#include +#include +#include +#include "zstd_zlibwrapper.h" + +#define LOG_FITBLK(...) /*printf(__VA_ARGS__)*/ +#define local static + +/* print nastygram and leave */ +local void quit(char *why) +{ + fprintf(stderr, "fitblk abort: %s\n", why); + exit(1); +} + +#define RAWLEN 4096 /* intermediate uncompressed buffer size */ + +/* compress from file to def until provided buffer is full or end of + input reached; return last deflate() return value, or Z_ERRNO if + there was read error on the file */ +local int partcompress(FILE *in, z_streamp def) +{ + int ret, flush; + unsigned char raw[RAWLEN]; + + flush = Z_SYNC_FLUSH; + do { + def->avail_in = (uInt)fread(raw, 1, RAWLEN, in); + if (ferror(in)) + return Z_ERRNO; + def->next_in = raw; + if (feof(in)) + flush = Z_FINISH; + LOG_FITBLK("partcompress1 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); + ret = deflate(def, flush); + LOG_FITBLK("partcompress2 ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); + assert(ret != Z_STREAM_ERROR); + } while (def->avail_out != 0 && flush == Z_SYNC_FLUSH); + return ret; +} + +/* recompress from inf's input to def's output; the input for inf and + the output for def are set in those structures before calling; + return last deflate() return value, or Z_MEM_ERROR if inflate() + was not able to allocate enough memory when it needed to */ +local int recompress(z_streamp inf, z_streamp def) +{ + int ret, flush; + unsigned char raw[RAWLEN]; + + flush = Z_NO_FLUSH; + LOG_FITBLK("recompress start\n"); + do { + /* decompress */ + inf->avail_out = RAWLEN; + inf->next_out = raw; + LOG_FITBLK("recompress1inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out); + ret = inflate(inf, Z_NO_FLUSH); + LOG_FITBLK("recompress2inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out); + assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR && + ret != Z_NEED_DICT); + if (ret == Z_MEM_ERROR) + return ret; + + /* compress what was decompresed until done or no room */ + def->avail_in = RAWLEN - inf->avail_out; + def->next_in = raw; + if (inf->avail_out != 0) + flush = Z_FINISH; + LOG_FITBLK("recompress1deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); + ret = deflate(def, flush); + LOG_FITBLK("recompress2deflate ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); + assert(ret != Z_STREAM_ERROR); + } while (ret != Z_STREAM_END && def->avail_out != 0); + return ret; +} + +#define EXCESS 256 /* empirically determined stream overage */ +#define MARGIN 8 /* amount to back off for completion */ + +/* compress from stdin to fixed-size block on stdout */ +int main(int argc, char **argv) +{ + int ret; /* return code */ + unsigned size; /* requested fixed output block size */ + unsigned have; /* bytes written by deflate() call */ + unsigned char *blk; /* intermediate and final stream */ + unsigned char *tmp; /* close to desired size stream */ + z_stream def, inf; /* zlib deflate and inflate states */ + + /* get requested output size */ + if (argc != 2) + quit("need one argument: size of output block"); + ret = (int)strtol(argv[1], argv + 1, 10); + if (argv[1][0] != 0) + quit("argument must be a number"); + if (ret < 8) /* 8 is minimum zlib stream size */ + quit("need positive size of 8 or greater"); + size = (unsigned)ret; + + printf("zlib version %s\n", ZLIB_VERSION); + if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); + + /* allocate memory for buffers and compression engine */ + blk = malloc(size + EXCESS); + def.zalloc = Z_NULL; + def.zfree = Z_NULL; + def.opaque = Z_NULL; + ret = deflateInit(&def, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK || blk == NULL) + quit("out of memory"); + + /* compress from stdin until output full, or no more input */ + def.avail_out = size + EXCESS; + def.next_out = blk; + LOG_FITBLK("partcompress1 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out); + ret = partcompress(stdin, &def); + printf("partcompress total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out); + if (ret == Z_ERRNO) + quit("error reading input"); + + /* if it all fit, then size was undersubscribed -- done! */ + if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { + /* write block to stdout */ + have = size + EXCESS - def.avail_out; + // if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + // quit("error writing output"); + + /* clean up and print results to stderr */ + ret = deflateEnd(&def); + assert(ret != Z_STREAM_ERROR); + free(blk); + fprintf(stderr, + "%u bytes unused out of %u requested (all input)\n", + size - have, size); + return 0; + } + + /* it didn't all fit -- set up for recompression */ + inf.zalloc = Z_NULL; + inf.zfree = Z_NULL; + inf.opaque = Z_NULL; + inf.avail_in = 0; + inf.next_in = Z_NULL; + ret = inflateInit(&inf); + tmp = malloc(size + EXCESS); + if (ret != Z_OK || tmp == NULL) + quit("out of memory"); + ret = deflateReset(&def); + assert(ret != Z_STREAM_ERROR); + + /* do first recompression close to the right amount */ + inf.avail_in = size + EXCESS; + inf.next_in = blk; + def.avail_out = size + EXCESS; + def.next_out = tmp; + LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); + ret = recompress(&inf, &def); + LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); + if (ret == Z_MEM_ERROR) + quit("out of memory"); + + /* set up for next recompression */ + ret = inflateReset(&inf); + assert(ret != Z_STREAM_ERROR); + ret = deflateReset(&def); + assert(ret != Z_STREAM_ERROR); + + /* do second and final recompression (third compression) */ + inf.avail_in = size - MARGIN; /* assure stream will complete */ + inf.next_in = tmp; + def.avail_out = size; + def.next_out = blk; + LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); + ret = recompress(&inf, &def); + LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); + if (ret == Z_MEM_ERROR) + quit("out of memory"); + assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */ + + /* done -- write block to stdout */ + have = size - def.avail_out; +// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) +// quit("error writing output"); + + /* clean up and print results to stderr */ + free(tmp); + ret = inflateEnd(&inf); + assert(ret != Z_STREAM_ERROR); + ret = deflateEnd(&def); + assert(ret != Z_STREAM_ERROR); + free(blk); + fprintf(stderr, + "%u bytes unused out of %u requested (%lu input)\n", + size - have, size, def.total_in); + return 0; +} diff --git a/src/zstd/zlibWrapper/examples/zwrapbench.c b/src/zstd/zlibWrapper/examples/zwrapbench.c new file mode 100644 index 0000000000000000000000000000000000000000..61031b9de798e62f040a731e6c4c185b9aae33d3 --- /dev/null +++ b/src/zstd/zlibWrapper/examples/zwrapbench.c @@ -0,0 +1,1019 @@ +/* + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + + +/* ************************************* +* Includes +***************************************/ +#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ +#include /* malloc, free */ +#include /* memset */ +#include /* fprintf, fopen, ftello64 */ +#include /* clock_t, clock, CLOCKS_PER_SEC */ +#include /* toupper */ +#include /* errno */ + +#include "timefn.h" /* UTIL_time_t, UTIL_getTime, UTIL_clockSpanMicro, UTIL_waitForNextTick */ +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#include "datagen.h" /* RDG_genBuffer */ +#include "xxhash.h" + +#include "zstd_zlibwrapper.h" + + + +/*-************************************ +* Tuning parameters +**************************************/ +#ifndef ZSTDCLI_CLEVEL_DEFAULT +# define ZSTDCLI_CLEVEL_DEFAULT 3 +#endif + + +/*-************************************ +* Constants +**************************************/ +#define COMPRESSOR_NAME "Zstandard wrapper for zlib command line interface" +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "v" ZSTD_VERSION_STRING +#endif +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR + +#ifndef ZSTD_GIT_COMMIT +# define ZSTD_GIT_COMMIT_STRING "" +#else +# define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT) +#endif + +#define NBLOOPS 3 +#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ +#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ +#define COOLPERIOD_SEC 10 + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); + +static U32 g_compressibilityDefault = 50; + + +/* ************************************* +* console display +***************************************/ +#define DEFAULT_DISPLAY_LEVEL 2 +#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static int g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +static FILE* displayOut; + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ + { g_time = clock(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(displayOut); } } +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; + + +/* ************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} + + +/* ************************************* +* Benchmark Parameters +***************************************/ +static unsigned g_nbIterations = NBLOOPS; +static size_t g_blockSize = 0; +int g_additionalParam = 0; + +void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } + +void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } + +void BMK_SetNbIterations(unsigned nbLoops) +{ + g_nbIterations = nbLoops; + DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations); +} + +void BMK_SetBlockSize(size_t blockSize) +{ + g_blockSize = blockSize; + DISPLAYLEVEL(2, "using blocks of size %u KB \n", (unsigned)(blockSize>>10)); +} + + +/* ******************************************************** +* Bench functions +**********************************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + +typedef struct +{ + z_const char* srcPtr; + size_t srcSize; + char* cPtr; + size_t cRoom; + size_t cSize; + char* resPtr; + size_t resSize; +} blockParam_t; + +typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD, BMK_ZLIB_REUSE, BMK_ZWRAP_ZLIB_REUSE, BMK_ZWRAP_ZSTD_REUSE } BMK_compressor; + + +static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, + const char* displayName, int cLevel, + const size_t* fileSizes, U32 nbFiles, + const void* dictBuffer, size_t dictBufferSize, BMK_compressor compressor) +{ + size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; + size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles)); + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); + size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ + void* const compressedBuffer = malloc(maxCompressedSize); + void* const resultBuffer = malloc(srcSize); + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + U32 nbBlocks; + + /* checks */ + if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx) + EXM_THROW(31, "allocation error : not enough memory"); + + /* init */ + if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ + + /* Init blockTable data */ + { z_const char* srcPtr = (z_const char*)srcBuffer; + char* cPtr = (char*)compressedBuffer; + char* resPtr = (char*)resultBuffer; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb ACTIVEPERIOD_MICROSEC) { + DISPLAYLEVEL(2, "\rcooling down ... \r"); + UTIL_sleep(COOLPERIOD_SEC); + coolTime = UTIL_getTime(); + } + + /* Compression */ + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (unsigned)srcSize); + if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ + + UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_waitForNextTick(); + clockStart = UTIL_getTime(); + + if (!cCompleted) { /* still some time to do compression tests */ + U32 nbLoops = 0; + if (compressor == BMK_ZSTD) { + ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); + ZSTD_customMem const cmem = { NULL, NULL, NULL }; + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_auto, zparams.cParams, cmem); + if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); + + do { + U32 blockNb; + size_t rSize; + for (blockNb=0; blockNbmaxTime; + } } + + cSize = 0; + { U32 blockNb; for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s\r", + marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio, + (double)srcSize / fastestC ); + + (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ +#if 1 + /* Decompression */ + if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ + + UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_waitForNextTick(); + clockStart = UTIL_getTime(); + + if (!dCompleted) { + U32 nbLoops = 0; + if (compressor == BMK_ZSTD) { + ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize); + if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure"); + do { + unsigned blockNb; + for (blockNb=0; blockNbmaxTime; + } } + + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", + marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio, + (double)srcSize / fastestC, + (double)srcSize / fastestD ); + + /* CRC Checking */ + { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + if (crcOrig!=crcCheck) { + size_t u; + DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); + for (u=0; u u) break; + bacc += blockTable[segNb].srcSize; + } + pos = (U32)(u - bacc); + bNb = pos / (128 KB); + DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos); + break; + } + if (u==srcSize-1) { /* should never happen */ + DISPLAY("no difference detected\n"); + } } + break; + } } /* CRC Checking */ +#endif + } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ + + if (g_displayLevel == 1) { + double cSpeed = (double)srcSize / fastestC; + double dSpeed = (double)srcSize / fastestD; + if (g_additionalParam) + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam); + else + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); + } + DISPLAYLEVEL(2, "%2i#\n", cLevel); + } /* Bench */ + + /* clean up */ + free(blockTable); + free(compressedBuffer); + free(resultBuffer); + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + return 0; +} + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t const step = 64 MB; + BYTE* testmem = NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += step; + if (requiredMem > maxMemory) requiredMem = maxMemory; + + do { + testmem = (BYTE*)malloc((size_t)requiredMem); + requiredMem -= step; + } while (!testmem && requiredMem); /* do not allocate zero bytes */ + + free(testmem); + return (size_t)(requiredMem+1); /* avoid zero */ +} + +static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, + const char* displayName, int cLevel, int cLevelLast, + const size_t* fileSizes, unsigned nbFiles, + const void* dictBuffer, size_t dictBufferSize) +{ + int l; + + const char* pch = strrchr(displayName, '\\'); /* Windows */ + if (!pch) pch = strrchr(displayName, '/'); /* Linux */ + if (pch) displayName = pch+1; + + SET_REALTIME_PRIORITY; + + if (g_displayLevel == 1 && !g_additionalParam) + DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", + ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, + (unsigned)benchedSize, g_nbIterations, (unsigned)(g_blockSize>>10)); + + if (cLevelLast < cLevel) cLevelLast = cLevel; + + DISPLAY("benchmarking zstd %s (using ZSTD_CStream)\n", ZSTD_VERSION_STRING); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZSTD_STREAM); + } + + DISPLAY("benchmarking zstd %s (using ZSTD_CCtx)\n", ZSTD_VERSION_STRING); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZSTD); + } + + DISPLAY("benchmarking zstd %s (using zlibWrapper)\n", ZSTD_VERSION_STRING); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD_REUSE); + } + + DISPLAY("benchmarking zstd %s (zlibWrapper not reusing a context)\n", ZSTD_VERSION_STRING); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD); + } + + + if (cLevelLast > Z_BEST_COMPRESSION) cLevelLast = Z_BEST_COMPRESSION; + + DISPLAY("\n"); + DISPLAY("benchmarking zlib %s\n", ZLIB_VERSION); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZLIB_REUSE); + } + + DISPLAY("benchmarking zlib %s (zlib not reusing a context)\n", ZLIB_VERSION); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZLIB); + } + + DISPLAY("benchmarking zlib %s (using zlibWrapper)\n", ZLIB_VERSION); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB_REUSE); + } + + DISPLAY("benchmarking zlib %s (zlibWrapper not reusing a context)\n", ZLIB_VERSION); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB); + } +} + + +/*! BMK_loadFiles() : + Loads `buffer` with content of files listed within `fileNamesTable`. + At most, fills `buffer` entirely */ +static void BMK_loadFiles(void* buffer, size_t bufferSize, + size_t* fileSizes, + const char** fileNamesTable, unsigned nbFiles) +{ + size_t pos = 0, totalSize = 0; + unsigned n; + for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); + pos += readSize; } + fileSizes[n] = (size_t)fileSize; + totalSize += (size_t)fileSize; + fclose(f); + } + + if (totalSize == 0) EXM_THROW(12, "no data to bench"); +} + +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, + const char* dictFileName, int cLevel, int cLevelLast) +{ + void* srcBuffer; + size_t benchedSize; + void* dictBuffer = NULL; + size_t dictBufferSize = 0; + size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + char mfName[20] = {0}; + + if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); + + /* Load dictionary */ + if (dictFileName != NULL) { + U64 const dictFileSize = UTIL_getFileSize(dictFileName); + if (dictFileSize > 64 MB) + EXM_THROW(10, "dictionary file %s too large", dictFileName); + dictBufferSize = (size_t)dictFileSize; + dictBuffer = malloc(dictBufferSize); + if (dictBuffer==NULL) + EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (unsigned)dictBufferSize); + BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1); + } + + /* Memory allocation & restrictions */ + benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; + if (benchedSize < totalSizeToLoad) + DISPLAY("Not enough memory; testing %u MB only...\n", (unsigned)(benchedSize >> 20)); + srcBuffer = malloc(benchedSize + !benchedSize); + if (!srcBuffer) EXM_THROW(12, "not enough memory"); + + /* Load input buffer */ + BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); + + /* Bench */ + snprintf (mfName, sizeof(mfName), " %u files", nbFiles); + { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; + BMK_benchCLevel(srcBuffer, benchedSize, + displayName, cLevel, cLevelLast, + fileSizes, nbFiles, + dictBuffer, dictBufferSize); + } + + /* clean up */ + free(srcBuffer); + free(dictBuffer); + free(fileSizes); +} + + +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) +{ + char name[20] = {0}; + size_t benchedSize = 10000000; + void* const srcBuffer = malloc(benchedSize); + + /* Memory allocation */ + if (!srcBuffer) EXM_THROW(21, "not enough memory"); + + /* Fill input buffer */ + RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); + + /* Bench */ + snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); + + /* clean up */ + free(srcBuffer); +} + + +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, + const char* dictFileName, int cLevel, int cLevelLast) +{ + double const compressibility = (double)g_compressibilityDefault / 100; + + if (nbFiles == 0) + BMK_syntheticTest(cLevel, cLevelLast, compressibility); + else + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); + return 0; +} + + + + +/*-************************************ +* Command Line +**************************************/ +static int usage(const char* programName) +{ + DISPLAY(WELCOME_MESSAGE); + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName); + DISPLAY( "\n"); + DISPLAY( "FILE : a filename\n"); + DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -D file: use `file` as Dictionary \n"); + DISPLAY( " -h/-H : display help/long help and exit\n"); + DISPLAY( " -V : display Version number and exit\n"); + DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL); + DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); +#ifdef UTIL_HAS_CREATEFILELIST + DISPLAY( " -r : operate recursively on directories\n"); +#endif + DISPLAY( "\n"); + DISPLAY( "Benchmark arguments :\n"); + DISPLAY( " -b# : benchmark file(s), using # compression level (default : %d) \n", ZSTDCLI_CLEVEL_DEFAULT); + DISPLAY( " -e# : test all compression levels from -bX to # (default: %d)\n", ZSTDCLI_CLEVEL_DEFAULT); + DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); + DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); + return 0; +} + +static int badusage(const char* programName) +{ + DISPLAYLEVEL(1, "Incorrect parameters\n"); + if (g_displayLevel >= 1) usage(programName); + return 1; +} + +static void waitEnter(void) +{ + int unused; + DISPLAY("Press enter to continue...\n"); + unused = getchar(); + (void)unused; +} + +/*! readU32FromChar() : + @return : unsigned integer value reach from input in `char` format + Will also modify `*stringPtr`, advancing it to position where it stopped reading. + Note : this function can overflow if digit string > MAX_UINT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + return result; +} + + +#define CLEAN_RETURN(i) { operationResult = (i); goto _end; } + +int main(int argCount, char** argv) +{ + int argNb, + main_pause=0, + nextEntryIsDictionary=0, + operationResult=0, + nextArgumentIsFile=0; + int cLevel = ZSTDCLI_CLEVEL_DEFAULT; + int cLevelLast = 1; + unsigned recursive = 0; + const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ + unsigned filenameIdx = 0; + const char* programName = argv[0]; + const char* dictFileName = NULL; + char* dynNameSpace = NULL; +#ifdef UTIL_HAS_CREATEFILELIST + const char** fileNamesTable = NULL; + char* fileNamesBuf = NULL; + unsigned fileNamesNb; +#endif + + /* init */ + if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } + displayOut = stderr; + + /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ + { size_t pos; + for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } } + programName += pos; + } + + /* command switches */ + for(argNb=1; argNb='0') && (*argument<='9')) { + BMK_setAdditionalParam(readU32FromChar(&argument)); + } else + main_pause=1; + break; + /* unknown command */ + default : CLEAN_RETURN(badusage(programName)); + } + } + continue; + } /* if (argument[0]=='-') */ + + } /* if (nextArgumentIsAFile==0) */ + + if (nextEntryIsDictionary) { + nextEntryIsDictionary = 0; + dictFileName = argument; + continue; + } + + /* add filename to list */ + filenameTable[filenameIdx++] = argument; + } + + /* Welcome message (if verbose) */ + DISPLAYLEVEL(3, WELCOME_MESSAGE); + +#ifdef UTIL_HAS_CREATEFILELIST + if (recursive) { + fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, 1); + if (fileNamesTable) { + unsigned u; + for (u=0; u