Module pyaurorax.search.data_products

Use the AuroraX search engine to search and upload data product records.

Note that all functions and classes from submodules are all imported at this level of the data_products module. They can be referenced from here instead of digging in deeper to the submodules.

Sub-modules

pyaurorax.search.data_products.classes

Separated classes and functions used by the data_products module …

Classes

class DataProductsManager (aurorax_obj)

Expand source code

class DataProductsManager:
    """
    The DataProductsManager object is initialized within every PyAuroraX object. It acts as a way to access 
    the submodules and carry over configuration information in the super class.
    """

    __STANDARD_POLLING_SLEEP_TIME: float = 1.0  # Polling sleep time when waiting for data (after the initial sleep time), in seconds
    __UPLOAD_CHUNK_SIZE = 500  # number of ephemeris records to upload at a time

    def __init__(self, aurorax_obj):
        self.__aurorax_obj = aurorax_obj

    def search(self,
               start: datetime.datetime,
               end: datetime.datetime,
               programs: Optional[List[str]] = None,
               platforms: Optional[List[str]] = None,
               instrument_types: Optional[List[str]] = None,
               data_product_types: Optional[Literal["keogram", "montage", "movie", "summary_plot", "data_availability"]] = None,
               metadata_filters: Optional[Union[MetadataFilter, List[Dict]]] = None,
               metadata_filters_logical_operator: Optional[Literal["and", "or", "AND", "OR"]] = None,
               response_format: Optional[Dict] = None,
               poll_interval: float = __STANDARD_POLLING_SLEEP_TIME,
               return_immediately: bool = False,
               verbose: bool = False) -> DataProductSearch:
        """
        Search for data product records

        By default, this function will block and wait until the request completes and
        all data is downloaded. If you don't want to wait, set the 'return_immediately`
        value to True. The Search object will be returned right after the search has been
        started, and you can use the helper functions as part of that object to get the
        data when it's done.

        Note: At least one search criteria from programs, platforms, or instrument_types, 
        must be specified.

        Args:
            start (datetime.datetime): 
                Start timestamp of the search (inclusive)
         
            end (datetime.datetime): 
                End timestamp of the search (inclusive)
         
            programs (List[str]): 
                List of programs to search through, defaults to None
        
            platforms (List[str]): 
                List of platforms to search through, defaults to None
        
            instrument_types (List[str]): 
                List of instrument types to search through, defaults to None
        
            data_product_types (List[str]): 
                List of strings describing data product types to filter on e.g. "keogram", defaults 
                to None. Valid options are: `keogram`, `montage`, `movie`, `summary_plot`, and 
                `data_availability`.
        
            metadata_filters (MetadataFilter or List[Dict]): 
                The metadata filters to use when searching, defaults to None

            metadata_filters_logical_operator (str): 
                The logical operator to use when evaluating metadata filters (either `and` or `or`), 
                defaults to `and`. This parameter is deprecated in exchange for passing a 
                MetadataFilter object into the metadata_filters parameter. 
            
            response_format (Dict): 
                JSON representation of desired data response format
            
            poll_interval (float): 
                Time in seconds to wait between polling attempts, defaults to 1 second
            
            return_immediately (bool): 
                Initiate the search and return without waiting for data to be received, 
                defaults to False
            
            verbose (bool): 
                Output poll times and other progress messages, defaults to False

        Returns:
            A `pyaurorax.search.DataProductSearch` object
        """
        # show warnings
        if (isinstance(metadata_filters, MetadataFilter) and metadata_filters_logical_operator is not None):
            # logical operator supplied, but MetadataFilter supplied too
            show_warning("Supplying a MetadataFilter object in addition to the metadata_filters_logical_operator " +
                         "parameter is redundant. Only the MetadataFilter object is needed. The " +
                         "metadata_filters_logical_operator parameter will be ignored")

        # return
        return func_search(
            self.__aurorax_obj,
            start,
            end,
            programs,
            platforms,
            instrument_types,
            data_product_types,
            metadata_filters,
            metadata_filters_logical_operator,
            response_format,
            poll_interval,
            return_immediately,
            verbose,
        )

    def upload(
        self,
        identifier: int,
        records: List[DataProductData],
        validate_source: bool = False,
        chunk_size: int = __UPLOAD_CHUNK_SIZE,
    ) -> int:
        """
        Upload data product records to AuroraX

        Args:
            identifier (int): 
                The AuroraX data source ID

            records (List[DataProductData]): 
                Data product records to upload

            validate_source (bool): 
                Validate all records before uploading, defaults to False

            chunk_size (int): 
                Number of records to upload in a single call, defaults to 500

        Returns:
            0 for success, raises exception on error

        Raises:
            pyaurorax.exceptions.AuroraXUploadError: Upload error
            pyaurorax.exceptions.AuroraXError: Data source validation error
        """
        return func_upload(self.__aurorax_obj, identifier, records, validate_source, chunk_size)

    def delete_urls(self, data_source: DataSource, urls: List[str]) -> int:
        """
        Delete data products by URL.

        The API processes this request asynchronously, so this method will return
        immediately whether or not the data has already been deleted.

        Args:
            data_source (DataSource): 
                Data source associated with the data product records (note that
                identifier, program, platform, and instrument_type are required)

            urls (List[str]): 
                URLs of data product records to delete

        Returns:
            0 on success

        Raises:
            pyaurorax.exceptions.AuroraXAPIError: An API error was encountered
            pyaurorax.exceptions.AuroraXUnauthorizedError: Invalid API key for this operation
        """
        return func_delete_urls(self.__aurorax_obj, data_source, urls)

    def delete(self,
               data_source: DataSource,
               start: datetime.datetime,
               end: datetime.datetime,
               data_product_types: Optional[List[str]] = None) -> int:
        """
        Delete data products associated with a data source within a date range.

        The API processes this request asynchronously, so this method will return
        immediately whether or not the data has already been deleted.

        Args:
            data_source (DataSource): 
                Data source associated with the data product records (note that
                identifier, program, platform, and instrument_type are required)
            
            start (datetime.datetime): 
                Timestamp marking beginning of range to delete records for, inclusive
            
            end (datetime.datetime): 
                Timestamp marking end of range to delete records for, inclusive
            
            data_product_types (List[str]): 
                Specific types of data product to delete, e.g. ["keogram", "movie"]. If 
                omitted, all data product types will be deleted.

        Returns:
            1 on success

        Raises:
            pyaurorax.exceptions.AuroraXNotFoundError: Source not found
            pyaurorax.exceptions.AuroraXUnauthorizedError: Invalid API key for this operation
        """
        return func_delete(self.__aurorax_obj, data_source, start, end, data_product_types)

    def describe(self, search_obj: Optional[DataProductSearch] = None, query_dict: Optional[Dict] = None) -> str:
        """
        Describe a data product search as an "SQL-like" string. Either a DataProductSearch
        object can be supplied, or a dictionary of the raw JSON query.

        Args:
            search_obj (DataProductSearch): 
                The data product search to describe, optional

            query_dict (Dict): 
                The data product search query represented as a raw dictionary, optional

        Returns:
            The "SQL-like" string describing the data product search object
        """
        return func_describe(self.__aurorax_obj, search_obj, query_dict)

    def get_request_url(self, request_id: str) -> str:
        """
        Get the data product search request URL for a given
        request ID. This URL can be used for subsequent
        pyaurorax.requests function calls. Primarily this method
        facilitates delving into details about a set of already-submitted
        data product searches.

        Args:
            request_id (str): 
                The request identifier

        Returns:
            The request URL
        """
        return func_get_request_url(self.__aurorax_obj, request_id)

The DataProductsManager object is initialized within every PyAuroraX object. It acts as a way to access the submodules and carry over configuration information in the super class.

Methods

def delete(self,
data_source: DataSource,
start: datetime.datetime,
end: datetime.datetime,
data_product_types: List[str] | None = None) ‑> int

Expand source code

def delete(self,
           data_source: DataSource,
           start: datetime.datetime,
           end: datetime.datetime,
           data_product_types: Optional[List[str]] = None) -> int:
    """
    Delete data products associated with a data source within a date range.

    The API processes this request asynchronously, so this method will return
    immediately whether or not the data has already been deleted.

    Args:
        data_source (DataSource): 
            Data source associated with the data product records (note that
            identifier, program, platform, and instrument_type are required)
        
        start (datetime.datetime): 
            Timestamp marking beginning of range to delete records for, inclusive
        
        end (datetime.datetime): 
            Timestamp marking end of range to delete records for, inclusive
        
        data_product_types (List[str]): 
            Specific types of data product to delete, e.g. ["keogram", "movie"]. If 
            omitted, all data product types will be deleted.

    Returns:
        1 on success

    Raises:
        pyaurorax.exceptions.AuroraXNotFoundError: Source not found
        pyaurorax.exceptions.AuroraXUnauthorizedError: Invalid API key for this operation
    """
    return func_delete(self.__aurorax_obj, data_source, start, end, data_product_types)

Delete data products associated with a data source within a date range.

The API processes this request asynchronously, so this method will return immediately whether or not the data has already been deleted.

Args

data_source : DataSource

Data source associated with the data product records (note that identifier, program, platform, and instrument_type are required)

start : datetime.datetime

Timestamp marking beginning of range to delete records for, inclusive

end : datetime.datetime

Timestamp marking end of range to delete records for, inclusive

data_product_types : List[str]

Specific types of data product to delete, e.g. ["keogram", "movie"]. If omitted, all data product types will be deleted.

Returns

1 on success

Raises

AuroraXNotFoundError

Source not found

AuroraXUnauthorizedError

Invalid API key for this operation

def delete_urls(self,
data_source: DataSource,
urls: List[str]) ‑> int

Expand source code

def delete_urls(self, data_source: DataSource, urls: List[str]) -> int:
    """
    Delete data products by URL.

    The API processes this request asynchronously, so this method will return
    immediately whether or not the data has already been deleted.

    Args:
        data_source (DataSource): 
            Data source associated with the data product records (note that
            identifier, program, platform, and instrument_type are required)

        urls (List[str]): 
            URLs of data product records to delete

    Returns:
        0 on success

    Raises:
        pyaurorax.exceptions.AuroraXAPIError: An API error was encountered
        pyaurorax.exceptions.AuroraXUnauthorizedError: Invalid API key for this operation
    """
    return func_delete_urls(self.__aurorax_obj, data_source, urls)

Delete data products by URL.

The API processes this request asynchronously, so this method will return immediately whether or not the data has already been deleted.

Args

data_source : DataSource

Data source associated with the data product records (note that identifier, program, platform, and instrument_type are required)

urls : List[str]

URLs of data product records to delete

Returns

0 on success

Raises

AuroraXAPIError

An API error was encountered

AuroraXUnauthorizedError

Invalid API key for this operation

def describe(self,
search_obj: DataProductSearch | None = None,
query_dict: Dict | None = None) ‑> str

Expand source code

def describe(self, search_obj: Optional[DataProductSearch] = None, query_dict: Optional[Dict] = None) -> str:
    """
    Describe a data product search as an "SQL-like" string. Either a DataProductSearch
    object can be supplied, or a dictionary of the raw JSON query.

    Args:
        search_obj (DataProductSearch): 
            The data product search to describe, optional

        query_dict (Dict): 
            The data product search query represented as a raw dictionary, optional

    Returns:
        The "SQL-like" string describing the data product search object
    """
    return func_describe(self.__aurorax_obj, search_obj, query_dict)

Describe a data product search as an "SQL-like" string. Either a DataProductSearch object can be supplied, or a dictionary of the raw JSON query.

Args

search_obj : DataProductSearch

The data product search to describe, optional

query_dict : Dict

The data product search query represented as a raw dictionary, optional

Returns

The "SQL-like" string describing the data product search object

def get_request_url(self, request_id: str) ‑> str

Expand source code

def get_request_url(self, request_id: str) -> str:
    """
    Get the data product search request URL for a given
    request ID. This URL can be used for subsequent
    pyaurorax.requests function calls. Primarily this method
    facilitates delving into details about a set of already-submitted
    data product searches.

    Args:
        request_id (str): 
            The request identifier

    Returns:
        The request URL
    """
    return func_get_request_url(self.__aurorax_obj, request_id)

Get the data product search request URL for a given request ID. This URL can be used for subsequent pyaurorax.requests function calls. Primarily this method facilitates delving into details about a set of already-submitted data product searches.

Args

request_id : str

The request identifier

Returns

The request URL

def search(self,
start: datetime.datetime,
end: datetime.datetime,
programs: List[str] | None = None,
platforms: List[str] | None = None,
instrument_types: List[str] | None = None,
data_product_types: Literal['keogram', 'montage', 'movie', 'summary_plot', 'data_availability'] | None = None,
metadata_filters: MetadataFilter | List[Dict] | None = None,
metadata_filters_logical_operator: Literal['and', 'or', 'AND', 'OR'] | None = None,
response_format: Dict | None = None,
poll_interval: float = 1.0,
return_immediately: bool = False,
verbose: bool = False) ‑> DataProductSearch

Expand source code

def search(self,
           start: datetime.datetime,
           end: datetime.datetime,
           programs: Optional[List[str]] = None,
           platforms: Optional[List[str]] = None,
           instrument_types: Optional[List[str]] = None,
           data_product_types: Optional[Literal["keogram", "montage", "movie", "summary_plot", "data_availability"]] = None,
           metadata_filters: Optional[Union[MetadataFilter, List[Dict]]] = None,
           metadata_filters_logical_operator: Optional[Literal["and", "or", "AND", "OR"]] = None,
           response_format: Optional[Dict] = None,
           poll_interval: float = __STANDARD_POLLING_SLEEP_TIME,
           return_immediately: bool = False,
           verbose: bool = False) -> DataProductSearch:
    """
    Search for data product records

    By default, this function will block and wait until the request completes and
    all data is downloaded. If you don't want to wait, set the 'return_immediately`
    value to True. The Search object will be returned right after the search has been
    started, and you can use the helper functions as part of that object to get the
    data when it's done.

    Note: At least one search criteria from programs, platforms, or instrument_types, 
    must be specified.

    Args:
        start (datetime.datetime): 
            Start timestamp of the search (inclusive)
     
        end (datetime.datetime): 
            End timestamp of the search (inclusive)
     
        programs (List[str]): 
            List of programs to search through, defaults to None
    
        platforms (List[str]): 
            List of platforms to search through, defaults to None
    
        instrument_types (List[str]): 
            List of instrument types to search through, defaults to None
    
        data_product_types (List[str]): 
            List of strings describing data product types to filter on e.g. "keogram", defaults 
            to None. Valid options are: `keogram`, `montage`, `movie`, `summary_plot`, and 
            `data_availability`.
    
        metadata_filters (MetadataFilter or List[Dict]): 
            The metadata filters to use when searching, defaults to None

        metadata_filters_logical_operator (str): 
            The logical operator to use when evaluating metadata filters (either `and` or `or`), 
            defaults to `and`. This parameter is deprecated in exchange for passing a 
            MetadataFilter object into the metadata_filters parameter. 
        
        response_format (Dict): 
            JSON representation of desired data response format
        
        poll_interval (float): 
            Time in seconds to wait between polling attempts, defaults to 1 second
        
        return_immediately (bool): 
            Initiate the search and return without waiting for data to be received, 
            defaults to False
        
        verbose (bool): 
            Output poll times and other progress messages, defaults to False

    Returns:
        A `pyaurorax.search.DataProductSearch` object
    """
    # show warnings
    if (isinstance(metadata_filters, MetadataFilter) and metadata_filters_logical_operator is not None):
        # logical operator supplied, but MetadataFilter supplied too
        show_warning("Supplying a MetadataFilter object in addition to the metadata_filters_logical_operator " +
                     "parameter is redundant. Only the MetadataFilter object is needed. The " +
                     "metadata_filters_logical_operator parameter will be ignored")

    # return
    return func_search(
        self.__aurorax_obj,
        start,
        end,
        programs,
        platforms,
        instrument_types,
        data_product_types,
        metadata_filters,
        metadata_filters_logical_operator,
        response_format,
        poll_interval,
        return_immediately,
        verbose,
    )

Search for data product records

By default, this function will block and wait until the request completes and all data is downloaded. If you don't want to wait, set the 'return_immediately` value to True. The Search object will be returned right after the search has been started, and you can use the helper functions as part of that object to get the data when it's done.

Note: At least one search criteria from programs, platforms, or instrument_types, must be specified.

Args

start : datetime.datetime

Start timestamp of the search (inclusive)

end (datetime.datetime): End timestamp of the search (inclusive)

programs (List[str]): List of programs to search through, defaults to None

platforms (List[str]): List of platforms to search through, defaults to None

instrument_types (List[str]): List of instrument types to search through, defaults to None

data_product_types (List[str]): List of strings describing data product types to filter on e.g. "keogram", defaults to None. Valid options are: keogram, montage, movie, summary_plot, and data_availability.

metadata_filters (MetadataFilter or List[Dict]): The metadata filters to use when searching, defaults to None

metadata_filters_logical_operator (str): The logical operator to use when evaluating metadata filters (either and or or), defaults to and. This parameter is deprecated in exchange for passing a MetadataFilter object into the metadata_filters parameter.

response_format (Dict): JSON representation of desired data response format

poll_interval (float): Time in seconds to wait between polling attempts, defaults to 1 second

return_immediately (bool): Initiate the search and return without waiting for data to be received, defaults to False

verbose (bool): Output poll times and other progress messages, defaults to False

Returns

A DataProductSearch object

def upload(self,
identifier: int,
records: List[DataProductData],
validate_source: bool = False,
chunk_size: int = 500) ‑> int

Expand source code

def upload(
    self,
    identifier: int,
    records: List[DataProductData],
    validate_source: bool = False,
    chunk_size: int = __UPLOAD_CHUNK_SIZE,
) -> int:
    """
    Upload data product records to AuroraX

    Args:
        identifier (int): 
            The AuroraX data source ID

        records (List[DataProductData]): 
            Data product records to upload

        validate_source (bool): 
            Validate all records before uploading, defaults to False

        chunk_size (int): 
            Number of records to upload in a single call, defaults to 500

    Returns:
        0 for success, raises exception on error

    Raises:
        pyaurorax.exceptions.AuroraXUploadError: Upload error
        pyaurorax.exceptions.AuroraXError: Data source validation error
    """
    return func_upload(self.__aurorax_obj, identifier, records, validate_source, chunk_size)

Upload data product records to AuroraX

Args

identifier : int

The AuroraX data source ID

records : List[DataProductData]

Data product records to upload

validate_source : bool

Validate all records before uploading, defaults to False

chunk_size : int

Number of records to upload in a single call, defaults to 500

Returns

0 for success, raises exception on error

Raises

AuroraXUploadError

Upload error

AuroraXError

Data source validation error