# -*- coding: utf-8 -*- # MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) # 2020 MinIO, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Common request/response configuration of S3 APIs.""" # pylint: disable=invalid-name from __future__ import absolute_import, annotations from abc import ABC, abstractmethod from dataclasses import dataclass, field from datetime import datetime from typing import IO, Optional, Type, TypeVar, cast from xml.etree import ElementTree as ET from .error import MinioException from .helpers import quote from .sse import SseCustomerKey from .time import to_http_header from .xml import SubElement, find, findall, findtext COPY = "COPY" REPLACE = "REPLACE" DISABLED = "Disabled" ENABLED = "Enabled" GOVERNANCE = "GOVERNANCE" COMPLIANCE = "COMPLIANCE" _MAX_KEY_LENGTH = 128 _MAX_VALUE_LENGTH = 256 _MAX_OBJECT_TAG_COUNT = 10 _MAX_TAG_COUNT = 50 A = TypeVar("A", bound="Tags") class Tags(dict): """dict extended to bucket/object tags.""" def __init__(self, for_object: bool = False): self._for_object = for_object super().__init__() def __setitem__(self, key: str, value: str): limit = _MAX_OBJECT_TAG_COUNT if self._for_object else _MAX_TAG_COUNT if len(self) == limit: tag_type = "object" if self._for_object else "bucket" raise ValueError(f"only {limit} {tag_type} tags are allowed") if not key or len(key) > _MAX_KEY_LENGTH or "&" in key: raise ValueError(f"invalid tag key '{key}'") if value is None or len(value) > _MAX_VALUE_LENGTH or "&" in value: raise ValueError(f"invalid tag value '{value}'") super().__setitem__(key, value) @classmethod def new_bucket_tags(cls: Type[A]) -> A: """Create new bucket tags.""" return cls() @classmethod def new_object_tags(cls: Type[A]) -> A: """Create new object tags.""" return cls(True) @classmethod def fromxml(cls: Type[A], element: ET.Element) -> A: """Create new object with values from XML element.""" elements = findall(element, "Tag") obj = cls() for tag in elements: key = cast(str, findtext(tag, "Key", True)) value = cast(str, findtext(tag, "Value", True)) obj[key] = value return obj def toxml(self, element: Optional[ET.Element]) -> ET.Element: """Convert to XML.""" if element is None: raise ValueError("element must be provided") for key, value in self.items(): tag = SubElement(element, "Tag") SubElement(tag, "Key", key) SubElement(tag, "Value", value) return element B = TypeVar("B", bound="Tag") @dataclass(frozen=True) class Tag: """Tag.""" key: str value: str def __post_init__(self): if not self.key: raise ValueError("key must be provided") if self.value is None: raise ValueError("value must be provided") @classmethod def fromxml(cls: Type[B], element: ET.Element) -> B: """Create new object with values from XML element.""" element = cast(ET.Element, find(element, "Tag", True)) key = cast(str, findtext(element, "Key", True)) value = cast(str, findtext(element, "Value", True)) return cls(key, value) def toxml(self, element: Optional[ET.Element]) -> ET.Element: """Convert to XML.""" if element is None: raise ValueError("element must be provided") element = SubElement(element, "Tag") SubElement(element, "Key", self.key) SubElement(element, "Value", self.value) return element C = TypeVar("C", bound="AndOperator") @dataclass(frozen=True) class AndOperator: """AND operator.""" prefix: Optional[str] = None tags: Optional[Tags] = None def __post_init__(self): if self.prefix is None and not self.tags: raise ValueError("at least prefix or tags must be provided") @classmethod def fromxml(cls: Type[C], element: ET.Element) -> C: """Create new object with values from XML element.""" element = cast(ET.Element, find(element, "And", True)) prefix = findtext(element, "Prefix") tags = ( None if find(element, "Tag") is None else Tags.fromxml(element) ) return cls(prefix, tags) def toxml(self, element: Optional[ET.Element]) -> ET.Element: """Convert to XML.""" if element is None: raise ValueError("element must be provided") element = SubElement(element, "And") if self.prefix is not None: SubElement(element, "Prefix", self.prefix) if self.tags is not None: self.tags.toxml(element) return element D = TypeVar("D", bound="Filter") @dataclass(frozen=True) class Filter: """Lifecycle rule filter.""" and_operator: Optional[AndOperator] = None prefix: Optional[str] = None tag: Optional[Tag] = None def __post_init__(self): valid = ( (self.and_operator is not None) ^ (self.prefix is not None) ^ (self.tag is not None) ) if not valid: raise ValueError("only one of and, prefix or tag must be provided") @classmethod def fromxml(cls: Type[D], element: ET.Element) -> D: """Create new object with values from XML element.""" element = cast(ET.Element, find(element, "Filter", True)) and_operator = ( None if find(element, "And") is None else AndOperator.fromxml(element) ) prefix = findtext(element, "Prefix") tag = None if find(element, "Tag") is None else Tag.fromxml(element) return cls(and_operator, prefix, tag) def toxml(self, element: Optional[ET.Element]) -> ET.Element: """Convert to XML.""" if element is None: raise ValueError("element must be provided") element = SubElement(element, "Filter") if self.and_operator: self.and_operator.toxml(element) if self.prefix is not None: SubElement(element, "Prefix", self.prefix) if self.tag is not None: self.tag.toxml(element) return element @dataclass(frozen=True) class BaseRule(ABC): """Base rule class for Replication and Lifecycle.""" status: str rule_filter: Optional[Filter] = None rule_id: Optional[str] = None def __post_init__(self): check_status(self.status) if self.rule_id is not None: self.rule_id = self.rule_id.strip() if not self.rule_id: raise ValueError("rule ID must be non-empty string") if len(self.rule_id) > 255: raise ValueError("rule ID must not exceed 255 characters") @abstractmethod def _require_subclass_implementation(self) -> None: """Dummy abstract method to enforce abstract class behavior.""" @staticmethod def parsexml( element: ET.Element, ) -> tuple[str, Optional[Filter], Optional[str]]: """Parse XML and return filter and ID.""" return ( cast(str, findtext(element, "Status", True)), ( None if find(element, "Filter") is None else Filter.fromxml(element) ), findtext(element, "ID"), ) def toxml(self, element: Optional[ET.Element]) -> ET.Element: """Convert to XML.""" if element is None: raise ValueError("element must be provided") SubElement(element, "Status", self.status) if self.rule_filter: self.rule_filter.toxml(element) if self.rule_id is not None: SubElement(element, "ID", self.rule_id) return element def check_status(status: str): """Validate status.""" if status not in [ENABLED, DISABLED]: raise ValueError("status must be 'Enabled' or 'Disabled'") @dataclass class ObjectConditionalReadArgs(ABC): """Base argument class holds condition properties for reading object.""" bucket_name: str object_name: str region: Optional[str] = None version_id: Optional[str] = None ssec: Optional[SseCustomerKey] = None offset: Optional[int] = None length: Optional[int] = None match_etag: Optional[str] = None not_match_etag: Optional[str] = None modified_since: Optional[datetime] = None unmodified_since: Optional[datetime] = None def __post_init__(self): if ( self.ssec is not None and not isinstance(self.ssec, SseCustomerKey) ): raise ValueError("ssec must be SseCustomerKey type") if self.offset is not None and self.offset < 0: raise ValueError("offset should be zero or greater") if self.length is not None and self.length <= 0: raise ValueError("length should be greater than zero") if self.match_etag is not None and self.match_etag == "": raise ValueError("match_etag must not be empty") if self.not_match_etag is not None and self.not_match_etag == "": raise ValueError("not_match_etag must not be empty") if ( self.modified_since is not None and not isinstance(self.modified_since, datetime) ): raise ValueError("modified_since must be datetime type") if ( self.unmodified_since is not None and not isinstance(self.unmodified_since, datetime) ): raise ValueError("unmodified_since must be datetime type") @abstractmethod def _require_subclass_implementation(self) -> None: """Dummy abstract method to enforce abstract class behavior.""" def gen_copy_headers(self) -> dict[str, str]: """Generate copy source headers.""" copy_source = quote("/" + self.bucket_name + "/" + self.object_name) if self.version_id: copy_source += "?versionId=" + quote(self.version_id) headers = {"x-amz-copy-source": copy_source} if self.ssec: headers.update(self.ssec.copy_headers()) if self.match_etag: headers["x-amz-copy-source-if-match"] = self.match_etag if self.not_match_etag: headers["x-amz-copy-source-if-none-match"] = self.not_match_etag if self.modified_since: headers["x-amz-copy-source-if-modified-since"] = ( to_http_header(self.modified_since) ) if self.unmodified_since: headers["x-amz-copy-source-if-unmodified-since"] = ( to_http_header(self.unmodified_since) ) return headers E = TypeVar("E", bound="CopySource") @dataclass class CopySource(ObjectConditionalReadArgs): """A source object definition for copy_object method.""" def _require_subclass_implementation(self) -> None: """Dummy abstract method to enforce abstract class behavior.""" @classmethod def of(cls: Type[E], src: ObjectConditionalReadArgs) -> E: """Create CopySource from another source.""" return cls( bucket_name=src.bucket_name, object_name=src.object_name, region=src.region, version_id=src.version_id, ssec=src.ssec, offset=src.offset, length=src.length, match_etag=src.match_etag, not_match_etag=src.not_match_etag, modified_since=src.modified_since, unmodified_since=src.unmodified_since, ) F = TypeVar("F", bound="ComposeSource") @dataclass class ComposeSource(ObjectConditionalReadArgs): """A source object definition for compose_object method.""" _object_size: Optional[int] = field(default=None, init=False) _headers: Optional[dict[str, str]] = field(default=None, init=False) def _require_subclass_implementation(self) -> None: """Dummy abstract method to enforce abstract class behavior.""" def _validate_size(self, object_size: int): """Validate object size with offset and length.""" def make_error(name, value): ver = ("?versionId="+self.version_id) if self.version_id else "" return ValueError( f"Source {self.bucket_name}/{self.object_name}{ver}: " f"{name} {value} is beyond object size {object_size}" ) if self.offset is not None and self.offset >= object_size: raise make_error("offset", self.offset) if self.length is not None: if self.length > object_size: raise make_error("length", self.length) offset = self.offset or 0 if offset+self.length > object_size: raise make_error("compose size", offset+self.length) def build_headers(self, object_size: int, etag: str): """Build headers.""" self._validate_size(object_size) self._object_size = object_size headers = self.gen_copy_headers() headers["x-amz-copy-source-if-match"] = self.match_etag or etag self._headers = headers @property def object_size(self) -> Optional[int]: """Get object size.""" if self._object_size is None: raise MinioException( "build_headers() must be called prior to " "this method invocation", ) return self._object_size @property def headers(self) -> dict[str, str]: """Get headers.""" if self._headers is None: raise MinioException( "build_headers() must be called prior to " "this method invocation", ) return self._headers.copy() @classmethod def of(cls: Type[F], src: ObjectConditionalReadArgs) -> F: """Create ComposeSource from another source.""" return cls( bucket_name=src.bucket_name, object_name=src.object_name, region=src.region, version_id=src.version_id, ssec=src.ssec, offset=src.offset, length=src.length, match_etag=src.match_etag, not_match_etag=src.not_match_etag, modified_since=src.modified_since, unmodified_since=src.unmodified_since, ) @dataclass(frozen=True) class SnowballObject: """A source object definition for upload_snowball_objects method.""" object_name: str filename: Optional[str] = None data: Optional[IO[bytes]] = None length: Optional[int] = None mod_time: Optional[datetime] = None def __post_init__(self): if not (self.filename is not None) ^ (self.data is not None): raise ValueError("only one of filename or data must be provided") if self.data is not None and self.length is None: raise ValueError("length must be provided for data") if ( self.mod_time is not None and not isinstance(self.mod_time, datetime) ): raise ValueError("mod_time must be datetime type")