Module pyaurorax.data.ucalgary

Data downloading and reading routines for data provided by the University of Calgary.

# Copyright 2024 University of Calgary
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Data downloading and reading routines for data provided by the University of Calgary.
"""

import os
import datetime
from pathlib import Path
from typing import TYPE_CHECKING, Optional, List, Union, Literal
from pyucalgarysrs.data import (
    Observatory,
    Dataset,
    FileDownloadResult,
    FileListingResponse,
    Data,
    Skymap,
    Calibration,
)
from pyucalgarysrs.exceptions import SRSAPIError, SRSDownloadError
from ...exceptions import AuroraXAPIError, AuroraXDownloadError
from .read import ReadManager
if TYPE_CHECKING:
    from ...pyaurorax import PyAuroraX

__all__ = [
    "UCalgaryManager",
    "Observatory",
    "Dataset",
    "FileDownloadResult",
    "FileListingResponse",
    "Data",
    "Skymap",
    "Calibration",
]


class UCalgaryManager:
    """
    The UCalgaryManager object is initialized within every PyAuroraX object. It acts as a way to access 
    the submodules and carry over configuration information in the super class.
    """

    __DEFAULT_DOWNLOAD_N_PARALLEL = 5

    def __init__(self, aurorax_obj):
        self.__aurorax_obj: PyAuroraX = aurorax_obj

        # initialize sub-modules
        self.__readers = ReadManager(self.__aurorax_obj)

    @property
    def readers(self):
        """
        Access to the `read` submodule from within a PyAuroraX object.
        """
        return self.__readers

    def list_datasets(self, name: Optional[str] = None, timeout: Optional[int] = None) -> List[Dataset]:
        """
        List available datasets

        Args:
            name (str): 
                Supply a name used for filtering. If that name is found in the available dataset 
                names received from the API, it will be included in the results. This parameter is
                optional.
            
            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.
            
        Returns:
            A list of [`Dataset`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.Dataset)
            objects.
        
        Raises:
            pyaurorax.exceptions.AuroraXAPIError: An API error was encountered.
        """
        try:
            return self.__aurorax_obj.srs_obj.data.list_datasets(name=name, timeout=timeout)
        except SRSAPIError as e:
            raise AuroraXAPIError(e) from e

    def list_observatories(self,
                           instrument_array: Literal["themis_asi", "rego", "trex_rgb", "trex_nir", "trex_blue"],
                           uid: Optional[str] = None,
                           timeout: Optional[int] = None) -> List[Observatory]:
        """
        List information about observatories

        Args:
            instrument_array (str): 
                The instrument array to list observatories for. Valid values are: themis_asi, rego, 
                trex_rgb, trex_nir, and trex_blue.

            uid (str): 
                Supply a observatory unique identifier used for filtering (usually 4-letter site code). If that UID 
                is found in the available observatories received from the API, it will be included in the results. This 
                parameter is optional.
            
            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.
            
        Returns:
            A list of [`Observatory`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.Observatory)
            objects.
        
        Raises:
            pyaurorax.exceptions.AuroraXAPIError: An API error was encountered.
        """
        try:
            return self.__aurorax_obj.srs_obj.data.list_observatories(instrument_array, uid=uid, timeout=timeout)
        except SRSAPIError as e:
            raise AuroraXAPIError(e) from e

    def list_supported_read_datasets(self) -> List[str]:
        """
        List the datasets which have file reading capabilities supported.

        Returns:
            A list of the dataset names with file reading support.
        """
        return self.__aurorax_obj.srs_obj.data.list_supported_read_datasets()

    def is_read_supported(self, dataset_name: str) -> bool:
        """
        Check if a given dataset has file reading support. 
        
        Not all datasets available in the UCalgary Space Remote Sensing Open Data Platform 
        have special readfile routines in this library. This is because some datasets are 
        in basic formats such as JPG or PNG, so unique functions aren't necessary. We leave 
        it up to the user to open these basic files in whichever way they prefer. Use the 
        `list_supported_read_datasets()` function to see all datasets that have special
        file reading functionality in this library.

        Args:
            dataset_name (str): 
                The dataset name to check if file reading is supported. This parameter 
                is required.
        
        Returns:
            Boolean indicating if file reading is supported.
        """
        return self.__aurorax_obj.srs_obj.data.is_read_supported(dataset_name)

    def download(self,
                 dataset_name: str,
                 start: datetime.datetime,
                 end: datetime.datetime,
                 site_uid: Optional[str] = None,
                 device_uid: Optional[str] = None,
                 n_parallel: int = __DEFAULT_DOWNLOAD_N_PARALLEL,
                 overwrite: bool = False,
                 progress_bar_disable: bool = False,
                 progress_bar_ncols: Optional[int] = None,
                 progress_bar_ascii: Optional[str] = None,
                 progress_bar_desc: Optional[str] = None,
                 timeout: Optional[int] = None) -> FileDownloadResult:
        """
        Download data from the UCalgary Space Remote Sensing Open Data Platform.

        The parameters `dataset_name`, `start`, and `end` are required. All other parameters
        are optional.

        Note that usage of the site and device UID filters applies differently to some datasets.
        For example, both fields can be used for most raw and keogram data, but only site UID can
        be used for skymap datasets, and only device UID can be used for calibration datasets. If 
        fields are specified during a call in which site or device UID is not used, a UserWarning
        is display to provide the user with feedback about this detail.

        Args:
            dataset_name (str): 
                Name of the dataset to download data for. Use the `list_datasets()` function
                to get the possible values for this parameter. One example is "THEMIS_ASI_RAW". 
                Note that dataset names are case sensitive. This parameter is required.

            start (datetime.datetime): 
                Start timestamp to use (inclusive), expected to be in UTC. Any timezone data 
                will be ignored. This parameter is required.

            end (datetime.datetime): 
                End timestamp to use (inclusive), expected to be in UTC. Any timezone data 
                will be ignored. This parameter is required.

            site_uid (str): 
                The site UID to filter for. If specified, data will be downloaded for only the 
                site matching the given value. If excluded, data for all available sites will 
                be downloaded. An example value could be 'atha', meaning all data from the 
                Athabasca observatory will be downloaded for the given dataset name, start, and 
                end times. This parameter is optional.

            device_uid (str): 
                The device UID to filter for. If specified, data will be downloaded for only the
                device matching the given value. If excluded, data for all available devices will
                be downloaded. An example value could be 'themis02', meaning all data matching that
                device will be downloaded for the given dataset name, start, and end times. This
                parameter is optional.

            n_parallel (int): 
                Number of data files to download in parallel. Default value is 5. Adjust as needed 
                for your internet connection. This parameter is optional.

            overwrite (bool): 
                By default, data will not be re-downloaded if it already exists locally. Use 
                the `overwrite` parameter to force re-downloading. Default is `False`. This 
                parameter is optional.

            progress_bar_disable (bool): 
                Disable the progress bar. Default is `False`. This parameter is optional.

            progress_bar_ncols (int): 
                Number of columns for the progress bar (straight passthrough of the `ncols` 
                parameter in a tqdm progress bar). This parameter is optional. See Notes section
                below for further information.
            
            progress_bar_ascii (str): 
                ASCII value to use when constructing the visual aspect of the progress bar (straight 
                passthrough of the `ascii` parameter in a tqdm progress bar). This parameter is 
                optional. See Notes section below for further details.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.

        Returns:
            A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
            object containing details about what data files were downloaded.

        Raises:
            pyaurorax.exceptions.AuroraXDownloadError: an error was encountered while downloading a 
                specific file
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered

        Notes:
        --------
        The `progress_bar_*` parameters can be used to enable/disable/adjust the progress bar. 
        Excluding the `progress_bar_disable` parameter, all others are straight pass-throughs 
        to the tqdm progress bar function. The `progress_bar_ncols` parameter allows for 
        adjusting the width. The `progress_bar_ascii` parameter allows for adjusting the appearance 
        of the progress bar. And the `progress_bar_desc` parameter allows for adjusting the 
        description at the beginning of the progress bar. Further details can be found on the
        [tqdm documentation](https://tqdm.github.io/docs/tqdm/#tqdm-objects).

        Data downloading will use the `download_data_root_path` variable within the super class'
        object ([`PyAuroraX`](../../index.html#pyaurorax.PyAuroraX)) to determine where to save data to. If 
        you'd like to change this path to somewhere else you can change that variable before your
        download() call, like so:

        ```python
        import pyaurorax
        aurorax = pyaurorax.PyAuroraX()
        aurorax.data_download_root_path = "some_new_path"
        aurorax.data.download(dataset_name, start, end)
        ```
        """
        try:
            return self.__aurorax_obj.srs_obj.data.download(
                dataset_name,
                start,
                end,
                site_uid=site_uid,
                device_uid=device_uid,
                n_parallel=n_parallel,
                overwrite=overwrite,
                progress_bar_disable=progress_bar_disable,
                progress_bar_ncols=progress_bar_ncols,
                progress_bar_ascii=progress_bar_ascii,
                progress_bar_desc=progress_bar_desc,
                timeout=timeout,
            )
        except SRSDownloadError as e:
            raise AuroraXDownloadError(e) from e
        except SRSAPIError as e:
            raise AuroraXAPIError(e) from e

    def download_using_urls(self,
                            file_listing_response: FileListingResponse,
                            n_parallel: int = __DEFAULT_DOWNLOAD_N_PARALLEL,
                            overwrite: bool = False,
                            progress_bar_disable: bool = False,
                            progress_bar_ncols: Optional[int] = None,
                            progress_bar_ascii: Optional[str] = None,
                            progress_bar_desc: Optional[str] = None,
                            timeout: Optional[int] = None) -> FileDownloadResult:
        """
        Download data from the UCalgary Space Remote Sensing Open Data Platform using 
        a FileListingResponse object. This would be used in cases where more customization 
        is needed than the generic `download()` function. 
        
        One example of using this function would start by using `get_urls()` to retrieve the
        list of URLs available for download, then further process this list to fewer files
        based on some other requirement (ie. time down-sampling such as one file per hour). 
        Lastly using this function to download the new custom set URLs.

        Args:
            file_listing_response (FileListingResponse): 
                A [`FileListingResponse`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileListingResponse) 
                object returned from a `get_urls()` call, which contains a list of URLs to download 
                for a specific dataset. This parameter is required.

            n_parallel (int): 
                Number of data files to download in parallel. Default value is 5. Adjust as needed 
                for your internet connection. This parameter is optional.

            overwrite (bool): 
                By default, data will not be re-downloaded if it already exists locally. Use 
                the `overwrite` parameter to force re-downloading. Default is `False`. This 
                parameter is optional.

            progress_bar_disable (bool): 
                Disable the progress bar. Default is `False`. This parameter is optional.

            progress_bar_ncols (int): 
                Number of columns for the progress bar (straight passthrough of the `ncols` 
                parameter in a tqdm progress bar). This parameter is optional. See Notes section
                below for further information.
            
            progress_bar_ascii (str): 
                ASCII value to use when constructing the visual aspect of the progress bar (straight 
                passthrough of the `ascii` parameter in a tqdm progress bar). This parameter is 
                optional. See Notes section below for further details.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.

        Returns:
            A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
            object containing details about what data files were downloaded.

        Raises:
            pyaurorax.exceptions.AuroraXDownloadError: an error was encountered while downloading a 
                specific file
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered

        Notes:
        --------
        The `progress_bar_*` parameters can be used to enable/disable/adjust the progress bar. 
        Excluding the `progress_bar_disable` parameter, all others are straight pass-throughs 
        to the tqdm progress bar function. The `progress_bar_ncols` parameter allows for 
        adjusting the width. The `progress_bar_ascii` parameter allows for adjusting the appearance 
        of the progress bar. And the `progress_bar_desc` parameter allows for adjusting the 
        description at the beginning of the progress bar. Further details can be found on the
        [tqdm documentation](https://tqdm.github.io/docs/tqdm/#tqdm-objects).

        Data downloading will use the `download_data_root_path` variable within the super class'
        object ([`PyAuroraX`](../../index.html#pyaurorax.PyAuroraX)) to determine where to save data to. If 
        you'd like to change this path to somewhere else you can change that variable before your
        download() call, like so:

        ```python
        import pyaurorax
        aurorax = pyaurorax.PyAuroraX()
        aurorax.data_download_root_path = "some_new_path"
        aurorax.data.download(dataset_name, start, end)
        ```
        """
        try:
            return self.__aurorax_obj.srs_obj.data.download_using_urls(
                file_listing_response,
                n_parallel=n_parallel,
                overwrite=overwrite,
                progress_bar_disable=progress_bar_disable,
                progress_bar_ncols=progress_bar_ncols,
                progress_bar_ascii=progress_bar_ascii,
                progress_bar_desc=progress_bar_desc,
                timeout=timeout,
            )
        except SRSDownloadError as e:
            raise AuroraXDownloadError(e) from e
        except SRSAPIError as e:
            raise AuroraXAPIError(e) from e

    def get_urls(self,
                 dataset_name: str,
                 start: datetime.datetime,
                 end: datetime.datetime,
                 site_uid: Optional[str] = None,
                 device_uid: Optional[str] = None,
                 timeout: Optional[int] = None) -> FileListingResponse:
        """
        Get URLs of data files

        The parameters `dataset_name`, `start`, and `end` are required. All other parameters
        are optional.

        Note that usage of the site and device UID filters applies differently to some datasets.
        For example, both fields can be used for most raw and keogram data, but only site UID can
        be used for skymap datasets, and only device UID can be used for calibration datasets. If 
        fields are specified during a call in which site or device UID is not used, a UserWarning
        is display to provide the user with feedback about this detail.

        Args:
            dataset_name (str): 
                Name of the dataset to download data for. Use the `list_datasets()` function
                to get the possible values for this parameter. One example is "THEMIS_ASI_RAW". 
                Note that dataset names are case sensitive. This parameter is required.

            start (datetime.datetime): 
                Start timestamp to use (inclusive), expected to be in UTC. Any timezone data 
                will be ignored. This parameter is required.

            end (datetime.datetime): 
                End timestamp to use (inclusive), expected to be in UTC. Any timezone data 
                will be ignored. This parameter is required.

            site_uid (str): 
                The site UID to filter for. If specified, data will be downloaded for only the 
                site matching the given value. If excluded, data for all available sites will 
                be downloaded. An example value could be 'atha', meaning all data from the 
                Athabasca observatory will be downloaded for the given dataset name, start, and 
                end times. This parameter is optional.

            device_uid (str): 
                The device UID to filter for. If specified, data will be downloaded for only the
                device matching the given value. If excluded, data for all available devices will
                be downloaded. An example value could be 'themis02', meaning all data matching that
                device will be downloaded for the given dataset name, start, and end times. This
                parameter is optional.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.
    
        Returns:
            A [`FileListingResponse`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileListingResponse)
            object containing a list of the available URLs, among other values.

        Raises:
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered
        """
        try:
            return self.__aurorax_obj.srs_obj.data.get_urls(
                dataset_name,
                start,
                end,
                site_uid=site_uid,
                device_uid=device_uid,
                timeout=timeout,
            )
        except SRSAPIError as e:
            raise AuroraXAPIError(e) from e

    def read(self,
             dataset: Dataset,
             file_list: Union[List[str], List[Path], str, Path],
             n_parallel: int = 1,
             first_record: bool = False,
             no_metadata: bool = False,
             quiet: bool = False) -> Data:
        """
        Read in data files for a given dataset. Note that only one type of dataset's data
        should be read in using a single call.

        Args:
            dataset (Dataset): 
                The dataset object for which the files are associated with. This parameter is
                required.
            
            file_list (List[str], List[Path], str, Path): 
                The files to read in. Absolute paths are recommended, but not technically
                necessary. This can be a single string for a file, or a list of strings to read
                in multiple files. This parameter is required.

            n_parallel (int): 
                Number of data files to read in parallel using multiprocessing. Default value 
                is 1. Adjust according to your computer's available resources. This parameter 
                is optional.
            
            first_record (bool): 
                Only read in the first record in each file. This is the same as the first_frame
                parameter in the themis-imager-readfile and trex-imager-readfile libraries, and
                is a read optimization if you only need one image per minute, as opposed to the
                full temporal resolution of data (e.g., 3sec cadence). This parameter is optional.
            
            no_metadata (bool): 
                Skip reading of metadata. This is a minor optimization if the metadata is not needed.
                Default is `False`. This parameter is optional.
            
            quiet (bool): 
                Do not print out errors while reading data files, if any are encountered. Any files
                that encounter errors will be, as usual, accessible via the `problematic_files` 
                attribute of the returned `Data` object. This parameter is optional.
        
        Returns:
            A [`Data`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.Data) 
            object containing the data read in, among other values.
        
        Raises:
            pyaurorax.exceptions.AuroraXUnsupportedReadError: an unsupported dataset was used when
                trying to read files.
            pyaurorax.exceptions.AuroraXError: a generic read error was encountered

        Notes:
        ---------
        For users who are familiar with the themis-imager-readfile and trex-imager-readfile
        libraries, the read function provides a near-identical usage. Further improvements have 
        been integrated, and those libraries are anticipated to be deprecated at some point in the
        future.
        """
        # NOTE: we do not wrap the exceptions here, instead we pass the call along
        # to the ReadManager object since the method and exception catching is
        # implemented there. No need to duplicate the exception handling logic.
        return self.__readers.read(
            dataset,
            file_list,
            n_parallel=n_parallel,
            first_record=first_record,
            no_metadata=no_metadata,
            quiet=quiet,
        )

    def download_best_skymap(
        self,
        dataset_name: str,
        site_uid: str,
        timestamp: datetime.datetime,
        timeout: Optional[int] = None,
        overwrite: bool = False,
    ) -> FileDownloadResult:
        """
        Download the skymap file that best matches the parameters supplied.

        Args:
            dataset_name (str): 
                Name of the dataset to download data for. Use the `list_datasets()` function
                to get the possible values for this parameter. One example is "THEMIS_ASI_SKYMAP_IDLSAV". 
                Note that dataset names are case sensitive. This parameter is required.

            site_uid (str): 
                The site UID to evaluate.

            timestamp (datetime.datetime): 
                The timestamp to use for deciding the best skymap, expected to be in UTC. Any timezone 
                data will be ignored. This parameter is required.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.

        Returns:
            A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
            object containing details about what data files were downloaded.

        Raises:
            ValueError: issue with supplied timestamp
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered        
        """
        # get list of all skymap urls for the dataset and site
        start_dt = datetime.datetime(2000, 1, 1)
        end_dt = datetime.datetime.now() + datetime.timedelta(days=5)
        file_listing_obj = self.get_urls(dataset_name, start_dt, end_dt, site_uid=site_uid, timeout=timeout)

        # filter down and find the best skymap for the timestamp supplied
        best_skymap_filename = None
        for url in file_listing_obj.urls:
            # extract start date for this skymap
            url_short = url.replace(file_listing_obj.path_prefix + "/", "")

            # parse filename into several values
            filename_split = os.path.basename(url_short).split('_')
            filename_times_split = filename_split[3].split('-')
            valid_interval_start_dt = datetime.datetime.strptime(filename_times_split[0], "%Y%m%d")

            # check start time
            if (timestamp >= valid_interval_start_dt):
                # valid
                #
                # NOTE: this works because of the order that the list is in already
                best_skymap_filename = url

        # check if we found a skymap
        if (best_skymap_filename is None):
            raise ValueError("Unable to determine a skymap recommendation")

        # set the filename
        file_listing_obj.urls = [best_skymap_filename]
        download_obj = self.download_using_urls(
            file_listing_obj,
            progress_bar_disable=True,
            overwrite=overwrite,
            timeout=timeout,
        )

        # return
        return download_obj

    def download_best_flatfield_calibration(
        self,
        dataset_name: str,
        device_uid: str,
        timestamp: datetime.datetime,
        timeout: Optional[int] = None,
        overwrite: bool = False,
    ) -> FileDownloadResult:
        """
        Download the flatfield calibration file that best matches the parameters supplied.

        Args:
            dataset_name (str): 
                Name of the dataset to download data for. Use the `list_datasets()` function
                to get the possible values for this parameter. One example is "THEMIS_ASI_SKYMAP_IDLSAV". 
                Note that dataset names are case sensitive. This parameter is required.

            device_uid (str): 
                The device UID to evaluate.

            timestamp (datetime.datetime): 
                The timestamp to use for deciding the best skymap, expected to be in UTC. Any timezone 
                data will be ignored. This parameter is required.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.

        Returns:
            A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
            object containing details about what data files were downloaded.

        Raises:
            ValueError: issue with supplied timestamp
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered        
        """
        # get list of all flatfield urls for the dataset and device
        start_dt = datetime.datetime(2000, 1, 1)
        end_dt = datetime.datetime.now() + datetime.timedelta(days=5)
        file_listing_obj = self.get_urls(dataset_name, start_dt, end_dt, device_uid=device_uid, timeout=timeout)

        # filter down and find the best skymap for the timestamp supplied
        best_cal_filename = None
        for url in file_listing_obj.urls:
            # extract start date for this skymap
            url_short = url.replace(file_listing_obj.path_prefix + "/", "")

            # parse filename into several values
            filename_split = os.path.basename(url_short).split('_')
            filename_times_split = filename_split[3].split('-')
            valid_interval_start_dt = datetime.datetime.strptime(filename_times_split[0], "%Y%m%d")

            # check start time
            if (timestamp >= valid_interval_start_dt):
                # valid
                #
                # NOTE: this works because of the order that the list is in already
                best_cal_filename = url

        # check if we found a skymap
        if (best_cal_filename is None):
            raise ValueError("Unable to determine a flatfield calibration recommendation")

        # set the filename
        file_listing_obj.urls = [best_cal_filename]
        download_obj = self.download_using_urls(
            file_listing_obj,
            progress_bar_disable=True,
            overwrite=overwrite,
            timeout=timeout,
        )

        # return
        return download_obj

    def download_best_rayleighs_calibration(
        self,
        dataset_name: str,
        device_uid: str,
        timestamp: datetime.datetime,
        timeout: Optional[int] = None,
        overwrite: bool = False,
    ) -> FileDownloadResult:
        """
        Download the Rayleighs calibration file that best matches the parameters supplied.

        Args:
            dataset_name (str): 
                Name of the dataset to download data for. Use the `list_datasets()` function
                to get the possible values for this parameter. One example is "REGO_CALIBRATION_RAYLEIGHS_IDLSAV". 
                Note that dataset names are case sensitive. This parameter is required.

            device_uid (str): 
                The device UID to evaluate.

            timestamp (datetime.datetime): 
                The timestamp to use for deciding the best calibration file, expected to be in 
                UTC. Any timezone data will be ignored. This parameter is required.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.

        Returns:
            A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
            object containing details about what data files were downloaded.

        Raises:
            ValueError: issue with supplied timestamp
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered        
        """
        # get list of all rayleighs urls for the dataset and device
        start_dt = datetime.datetime(2000, 1, 1)
        end_dt = datetime.datetime.now() + datetime.timedelta(days=5)
        file_listing_obj = self.get_urls(dataset_name, start_dt, end_dt, device_uid=device_uid, timeout=timeout)

        # filter down and find the best skymap for the timestamp supplied
        best_cal_filename = None
        for url in file_listing_obj.urls:
            # extract start date for this skymap
            url_short = url.replace(file_listing_obj.path_prefix + "/", "")

            # parse filename into several values
            filename_split = os.path.basename(url_short).split('_')
            filename_times_split = filename_split[3].split('-')
            valid_interval_start_dt = datetime.datetime.strptime(filename_times_split[0], "%Y%m%d")

            # check start time
            if (timestamp >= valid_interval_start_dt):
                # valid
                #
                # NOTE: this works because of the order that the list is in already
                best_cal_filename = url

        # check if we found a skymap
        if (best_cal_filename is None):
            raise ValueError("Unable to determine a Rayleighs calibration recommendation")

        # set the filename
        file_listing_obj.urls = [best_cal_filename]
        download_obj = self.download_using_urls(
            file_listing_obj,
            progress_bar_disable=True,
            overwrite=overwrite,
            timeout=timeout,
        )

        # return
        return download_obj

Sub-modules

pyaurorax.data.ucalgary.read

Classes

class Calibration (filename: str, detector_uid: str, version: str, generation_info: pyucalgarysrs.data.classes.CalibrationGenerationInfo, rayleighs_perdn_persecond: Optional[float] = None, flat_field_multiplier: Optional[numpy.ndarray] = None, dataset: Optional[pyucalgarysrs.data.classes.Dataset] = None)

Representation for a calibration file.

Attributes

filename : str

Filename for the calibration file, as an absolute path of its location on the local machine.

detector_uid : str

Detector/imager/camera unique identifier

version : str

Version number of the calibration file

generation_info : CalibrationGenerationInfo

Metadata describing details about this calibration's generation process

rayleighs_perdn_persecond : float

Calibrated value for Rayleighs per data number per second (R/dn/s). This value will be None if a flatfield calibration file was read instead of a rayleighs calibration file.

flat_field_multiplier : ndarray

Calibrated flat field array. This value will be None if a rayleighs calibration file was read instead of a flatfield calibration file.

dataset : Dataset

The Dataset object for this data.

Expand source code

@dataclass
class Calibration:
    """
    Representation for a calibration file.

    Attributes:
        filename (str): 
            Filename for the calibration file, as an absolute path of its location on the local machine.
        
        detector_uid (str): 
            Detector/imager/camera unique identifier
        
        version (str): 
            Version number of the calibration file
        
        generation_info (CalibrationGenerationInfo): 
            Metadata describing details about this calibration's generation process
        
        rayleighs_perdn_persecond (float): 
            Calibrated value for Rayleighs per data number per second (R/dn/s). This value will be None 
            if a flatfield calibration file was read instead of a rayleighs calibration file.
        
        flat_field_multiplier (ndarray): 
            Calibrated flat field array. This value will be None if a rayleighs calibration file was 
            read instead of a flatfield calibration file.
        
        dataset (Dataset): 
            The `Dataset` object for this data.
    """
    filename: str
    detector_uid: str
    version: str
    generation_info: CalibrationGenerationInfo
    rayleighs_perdn_persecond: Optional[float] = None
    flat_field_multiplier: Optional[ndarray] = None
    dataset: Optional[Dataset] = None

    def pretty_print(self):
        """
        A special print output for this class.
        """
        print("Calibration:")
        for var_name in dir(self):
            # exclude methods
            if (var_name.startswith("__") or var_name == "pretty_print"):
                continue

            # convert var to string format we want
            var_value = getattr(self, var_name)
            var_str = "None"
            if (var_name == "generation_info"):
                var_str = "CalibrationGenerationInfo(...)"
            elif (var_name == "dataset" and var_value is not None):
                var_str = "Dataset(...)"
            elif (var_value is not None):
                if (isinstance(var_value, ndarray)):
                    var_str = "array(dims=%s, dtype=%s)" % (var_value.shape, var_value.dtype)
                else:
                    var_str = str(var_value)

            # print string for this var
            print("  %-30s: %s" % (var_name, var_str))

Class variables

var dataset : Optional[pyucalgarysrs.data.classes.Dataset]

var detector_uid : str

var filename : str

var flat_field_multiplier : Optional[numpy.ndarray]

var generation_info : pyucalgarysrs.data.classes.CalibrationGenerationInfo

var rayleighs_perdn_persecond : Optional[float]

var version : str

Methods

def pretty_print(self)

A special print output for this class.

Expand source code

def pretty_print(self):
    """
    A special print output for this class.
    """
    print("Calibration:")
    for var_name in dir(self):
        # exclude methods
        if (var_name.startswith("__") or var_name == "pretty_print"):
            continue

        # convert var to string format we want
        var_value = getattr(self, var_name)
        var_str = "None"
        if (var_name == "generation_info"):
            var_str = "CalibrationGenerationInfo(...)"
        elif (var_name == "dataset" and var_value is not None):
            var_str = "Dataset(...)"
        elif (var_value is not None):
            if (isinstance(var_value, ndarray)):
                var_str = "array(dims=%s, dtype=%s)" % (var_value.shape, var_value.dtype)
            else:
                var_str = str(var_value)

        # print string for this var
        print("  %-30s: %s" % (var_name, var_str))

class Data (data: Any, timestamp: List[datetime.datetime], metadata: List[Dict], problematic_files: List[pyucalgarysrs.data.classes.ProblematicFile], calibrated_data: Any, dataset: Optional[pyucalgarysrs.data.classes.Dataset] = None)

Representation of the data read in from a pyaurorax.data.ucalgary.read call.

Attributes

data : Any

The loaded data. This can be one of the following types: ndarray, List[Skymap], List[Calibration].

timestamp : List[datetime.datetime]

List of timestamps for the read in data.

metadata : List[Dict]

List of dictionaries containing metadata specific to each timestamp/image/record.

problematic_files : List[ProblematicFiles]

A list detailing any files that encountered issues during reading.

calibrated_data : Any

A calibrated version of the data. Populated and utilized by data analysis libraries. Has a None value until calibrated data is inserted manually.

dataset : Dataset

The Dataset object for this data.

Expand source code

@dataclass
class Data:
    """
    Representation of the data read in from a `read` call.

    Attributes:
        data (Any): 
            The loaded data. This can be one of the following types: ndarray, List[Skymap], List[Calibration].
        
        timestamp (List[datetime.datetime]): 
            List of timestamps for the read in data.
        
        metadata (List[Dict]): 
            List of dictionaries containing metadata specific to each timestamp/image/record.
        
        problematic_files (List[ProblematicFiles]): 
            A list detailing any files that encountered issues during reading.
        
        calibrated_data (Any): 
            A calibrated version of the data. Populated and utilized by data analysis libraries. Has a `None` value
            until calibrated data is inserted manually.

        dataset (Dataset): 
            The `Dataset` object for this data.
    """
    data: Any
    timestamp: List[datetime.datetime]
    metadata: List[Dict]
    problematic_files: List[ProblematicFile]
    calibrated_data: Any
    dataset: Optional[Dataset] = None

    def __str__(self) -> str:
        return self.__repr__()

    def __repr__(self) -> str:
        # set data value
        if (isinstance(self.data, ndarray) is True):
            data_str = "array(dims=%s, dtype=%s)" % (self.data.shape, self.data.dtype)
        if (isinstance(self.data, GridData) is True):
            data_str = self.data.__repr__()
        elif (isinstance(self.data, list) is True):
            if (len(self.data) == 0):
                data_str = "[0 items]"
            elif (isinstance(self.data[0], Skymap) is True):
                if (len(self.data) == 1):
                    data_str = "[1 Skymap object]"
                else:
                    data_str = "[%d Skymap objects]" % (len(self.data))
            elif (isinstance(self.data[0], Calibration) is True):
                if (len(self.data) == 1):
                    data_str = "[1 Calibration object]"
                else:
                    data_str = "[%d Calibration objects]" % (len(self.data))
            else:
                data_str = "[%d items]" % (len(self.data))
        else:
            data_str = self.data.__repr__()

        # set timestamp string
        if (len(self.timestamp) == 0):
            timestamp_str = "[]"
        elif (len(self.timestamp) == 1):
            timestamp_str = "[1 datetime]"
        else:
            timestamp_str = "[%d datetimes]" % (len(self.timestamp))

        # set metadata string
        if (len(self.metadata) == 0):
            metadata_str = "[]"
        elif (len(self.metadata) == 1):
            metadata_str = "[1 dictionary]"
        else:
            metadata_str = "[%d dictionaries]" % (len(self.timestamp))

        # set rest of values
        problematic_files_str = "[]" if len(self.problematic_files) == 0 else "[%d problematic files]" % (len(self.problematic_files))
        calibrated_data_str = "None" if self.calibrated_data is None else "array(dims=%s, dtype=%s)" % (self.calibrated_data.shape,
                                                                                                        self.calibrated_data.dtype)
        dataset_str = "None" if self.dataset is None else self.dataset.__repr__()[0:75] + "...)"

        # return
        return "Data(data=%s, timestamp=%s, metadata=%s, problematic_files=%s, calibrated_data=%s, dataset=%s)" % (
            data_str,
            timestamp_str,
            metadata_str,
            problematic_files_str,
            calibrated_data_str,
            dataset_str,
        )

    def pretty_print(self):
        """
        A special print output for this class.
        """
        # set data value
        if (isinstance(self.data, ndarray) is True):
            data_str = "array(dims=%s, dtype=%s)" % (self.data.shape, self.data.dtype)
        elif (isinstance(self.data, list) is True):
            if (len(self.data) == 0):
                data_str = "[0 items]"
            elif (isinstance(self.data[0], Skymap) is True):
                if (len(self.data) == 1):
                    data_str = "[1 Skymap object]"
                else:
                    data_str = "[%d Skymap objects]" % (len(self.data))
            elif (isinstance(self.data[0], Calibration) is True):
                if (len(self.data) == 1):
                    data_str = "[1 Calibration object]"
                else:
                    data_str = "[%d Calibration objects]" % (len(self.data))
            else:
                data_str = "[%d items]" % (len(self.data))
        else:
            data_str = self.data.__repr__()

        # set timestamp string
        if (len(self.timestamp) == 0):
            timestamp_str = "[]"
        elif (len(self.timestamp) == 1):
            timestamp_str = "[1 datetime]"
        else:
            timestamp_str = "[%d datetimes]" % (len(self.timestamp))

        # set metadata string
        if (len(self.metadata) == 0):
            metadata_str = "[]"
        elif (len(self.metadata) == 1):
            metadata_str = "[1 dictionary]"
        else:
            metadata_str = "[%d dictionaries]" % (len(self.timestamp))

        # set rest of values
        problematic_files_str = "[]" if len(self.problematic_files) == 0 else "[%d problematic files]" % (len(self.problematic_files))
        calibrated_data_str = "None" if self.calibrated_data is None else "array(dims=%s, dtype=%s)" % (self.calibrated_data.shape,
                                                                                                        self.calibrated_data.dtype)
        dataset_str = "None" if self.dataset is None else self.dataset.__repr__()[0:75] + "...)"

        # print
        print("Data:")
        print("  %-22s: %s" % ("data", data_str))
        print("  %-22s: %s" % ("timestamp", timestamp_str))
        print("  %-22s: %s" % ("metadata", metadata_str))
        print("  %-22s: %s" % ("problematic_files", problematic_files_str))
        print("  %-22s: %s" % ("calibrated_data", calibrated_data_str))
        print("  %-22s: %s" % ("dataset", dataset_str))

Class variables

var calibrated_data : Any

var data : Any

var dataset : Optional[pyucalgarysrs.data.classes.Dataset]

var metadata : List[Dict]

var problematic_files : List[pyucalgarysrs.data.classes.ProblematicFile]

var timestamp : List[datetime.datetime]

Methods

def pretty_print(self)

A special print output for this class.

Expand source code

def pretty_print(self):
    """
    A special print output for this class.
    """
    # set data value
    if (isinstance(self.data, ndarray) is True):
        data_str = "array(dims=%s, dtype=%s)" % (self.data.shape, self.data.dtype)
    elif (isinstance(self.data, list) is True):
        if (len(self.data) == 0):
            data_str = "[0 items]"
        elif (isinstance(self.data[0], Skymap) is True):
            if (len(self.data) == 1):
                data_str = "[1 Skymap object]"
            else:
                data_str = "[%d Skymap objects]" % (len(self.data))
        elif (isinstance(self.data[0], Calibration) is True):
            if (len(self.data) == 1):
                data_str = "[1 Calibration object]"
            else:
                data_str = "[%d Calibration objects]" % (len(self.data))
        else:
            data_str = "[%d items]" % (len(self.data))
    else:
        data_str = self.data.__repr__()

    # set timestamp string
    if (len(self.timestamp) == 0):
        timestamp_str = "[]"
    elif (len(self.timestamp) == 1):
        timestamp_str = "[1 datetime]"
    else:
        timestamp_str = "[%d datetimes]" % (len(self.timestamp))

    # set metadata string
    if (len(self.metadata) == 0):
        metadata_str = "[]"
    elif (len(self.metadata) == 1):
        metadata_str = "[1 dictionary]"
    else:
        metadata_str = "[%d dictionaries]" % (len(self.timestamp))

    # set rest of values
    problematic_files_str = "[]" if len(self.problematic_files) == 0 else "[%d problematic files]" % (len(self.problematic_files))
    calibrated_data_str = "None" if self.calibrated_data is None else "array(dims=%s, dtype=%s)" % (self.calibrated_data.shape,
                                                                                                    self.calibrated_data.dtype)
    dataset_str = "None" if self.dataset is None else self.dataset.__repr__()[0:75] + "...)"

    # print
    print("Data:")
    print("  %-22s: %s" % ("data", data_str))
    print("  %-22s: %s" % ("timestamp", timestamp_str))
    print("  %-22s: %s" % ("metadata", metadata_str))
    print("  %-22s: %s" % ("problematic_files", problematic_files_str))
    print("  %-22s: %s" % ("calibrated_data", calibrated_data_str))
    print("  %-22s: %s" % ("dataset", dataset_str))

class Dataset (name: str, short_description: str, long_description: str, data_tree_url: str, file_listing_supported: bool, file_reading_supported: bool, level: str, doi: Optional[str] = None, doi_details: Optional[str] = None, citation: Optional[str] = None)

A dataset available from the UCalgary Space Remote Sensing API, with possibly support for downloading and/or reading.

Attributes

name : str

Dataset name

short_description : str

A short description about the dataset

long_description : str

A longer description about the dataset

data_tree_url : str

The data tree URL prefix. Used for saving data locally with a similar data tree structure compared to the UCalgary Open Data archive.

file_listing_supported : bool

Flag indicating if file listing (downloading) is supported for this dataset.

file_reading_supported : bool

Flag indicating if file reading is supported for this dataset.

level : str

Dataset level as per L0/L1/L2/etc standards.

doi : str

Dataset DOI unique identifier.

doi_details : str

Further details about the DOI.

citation : str

String to use when citing usage of the dataset.

provider : str

Data provider.

Expand source code

class Dataset:
    """
    A dataset available from the UCalgary Space Remote Sensing API, with possibly
    support for downloading and/or reading.

    Attributes:
        name (str): 
            Dataset name
        
        short_description (str): 
            A short description about the dataset
        
        long_description (str): 
            A longer description about the dataset
        
        data_tree_url (str): 
            The data tree URL prefix. Used for saving data locally with a similar data tree 
            structure compared to the UCalgary Open Data archive.
        
        file_listing_supported (bool): 
            Flag indicating if file listing (downloading) is supported for this dataset.
        
        file_reading_supported (bool): 
            Flag indicating if file reading is supported for this dataset.
        
        level (str): 
            Dataset level as per L0/L1/L2/etc standards.
        
        doi (str): 
            Dataset DOI unique identifier.
        
        doi_details (str): 
            Further details about the DOI.
        
        citation (str): 
            String to use when citing usage of the dataset.
        
        provider (str): 
            Data provider.
    """

    def __init__(self,
                 name: str,
                 short_description: str,
                 long_description: str,
                 data_tree_url: str,
                 file_listing_supported: bool,
                 file_reading_supported: bool,
                 level: str,
                 doi: Optional[str] = None,
                 doi_details: Optional[str] = None,
                 citation: Optional[str] = None):
        self.name = name
        self.short_description = short_description
        self.long_description = long_description
        self.data_tree_url = data_tree_url
        self.file_listing_supported = file_listing_supported
        self.file_reading_supported = file_reading_supported
        self.level = level
        self.doi = doi
        self.doi_details = doi_details
        self.citation = citation
        self.provider = "UCalgary"

    def __str__(self) -> str:
        return self.__repr__()

    def __repr__(self) -> str:
        return "Dataset(name=%s, short_description='%s', provider='%s', level='%s', doi_details='%s', ...)" % (
            self.name,
            self.short_description,
            self.provider,
            self.level,
            self.doi_details,
        )

    def pretty_print(self):
        """
        A special print output for this class.
        """
        print("Dataset:")
        for var_name in dir(self):
            # exclude methods
            if (var_name.startswith("__") or var_name == "pretty_print"):
                continue

            # convert var to string format we want
            var_value = getattr(self, var_name)
            print("  %-27s: %s" % (var_name, None if var_value is None else var_value))

Methods

def pretty_print(self)

A special print output for this class.

Expand source code

def pretty_print(self):
    """
    A special print output for this class.
    """
    print("Dataset:")
    for var_name in dir(self):
        # exclude methods
        if (var_name.startswith("__") or var_name == "pretty_print"):
            continue

        # convert var to string format we want
        var_value = getattr(self, var_name)
        print("  %-27s: %s" % (var_name, None if var_value is None else var_value))

class FileDownloadResult (filenames: List[str], count: int, total_bytes: int, output_root_path: str, dataset: pyucalgarysrs.data.classes.Dataset)

Representation of the results from a data download call.

Attributes

filenames : List[str]

List of downloaded files, as absolute paths of their location on the local machine.

count : int

Number of files downloaded

total_bytes : int

Cumulative amount of bytes saved on the local machine.

output_root_path : str

The root path of where the data was saved to on the local machine.

dataset : Dataset

The Dataset object for this data.

Expand source code

@dataclass
class FileDownloadResult:
    """
    Representation of the results from a data download call.

    Attributes:
        filenames (List[str]): 
            List of downloaded files, as absolute paths of their location on the local machine.
        
        count (int): 
            Number of files downloaded
        
        total_bytes (int): 
            Cumulative amount of bytes saved on the local machine.
        
        output_root_path (str): 
            The root path of where the data was saved to on the local machine.
        
        dataset (Dataset): 
            The `Dataset` object for this data.
    """
    filenames: List[str]
    count: int
    total_bytes: int
    output_root_path: str
    dataset: Dataset

Class variables

var count : int

var dataset : pyucalgarysrs.data.classes.Dataset

var filenames : List[str]

var output_root_path : str

var total_bytes : int

class FileListingResponse (urls: List[str], path_prefix: str, count: int, dataset: pyucalgarysrs.data.classes.Dataset, total_bytes: Optional[int] = None)

Representation of the file listing response from the UCalgary Space Remote Sensing API.

Attributes

urls : List[str]

A list of URLs for available data files.

path_prefix : str

The URL prefix, which is sed for saving data locally with a similar data tree structure compared to the UCalgary Open Data archive.

count : int

The number of URLs available.

dataset : Dataset

The Dataset object for this data.

total_bytes : int

The cumulative amount of bytes for the available URLs.

Expand source code

@dataclass
class FileListingResponse:
    """
    Representation of the file listing response from the UCalgary Space Remote Sensing API.

    Attributes:
        urls (List[str]): 
            A list of URLs for available data files.
        
        path_prefix (str): 
            The URL prefix, which is sed for saving data locally with a similar data tree 
            structure compared to the UCalgary Open Data archive.
        
        count (int): 
            The number of URLs available.
        
        dataset (Dataset): 
            The `Dataset` object for this data.
        
        total_bytes (int): 
            The cumulative amount of bytes for the available URLs.
    """
    urls: List[str]
    path_prefix: str
    count: int
    dataset: Dataset
    total_bytes: Optional[int] = None

Class variables

var count : int

var dataset : pyucalgarysrs.data.classes.Dataset

var path_prefix : str

var total_bytes : Optional[int]

var urls : List[str]

class Observatory (uid: str, full_name: str, geodetic_latitude: float, geodetic_longitude: float)

Representation for an observatory.

Attributes

uid : str

4-letter unique identifier (traditionally referred to as the site UID)

full_name : str

full location string for the observatory

geodetic_latitude : float

geodetic latitude for the observatory, in decimal format (-90 to 90)

geodetic_longitude : float

geodetic longitude for the observatory, in decimal format (-180 to 180)

provider : str

Data provider.

Expand source code

class Observatory:
    """
    Representation for an observatory.

    Attributes:
        uid (str): 
            4-letter unique identifier (traditionally referred to as the site UID)

        full_name (str): 
            full location string for the observatory
        
        geodetic_latitude (float): 
            geodetic latitude for the observatory, in decimal format (-90 to 90)
        
        geodetic_longitude (float): 
            geodetic longitude for the observatory, in decimal format (-180 to 180)

        provider (str): 
            Data provider.
    """

    def __init__(self, uid: str, full_name: str, geodetic_latitude: float, geodetic_longitude: float):
        self.uid = uid
        self.full_name = full_name
        self.geodetic_latitude = geodetic_latitude
        self.geodetic_longitude = geodetic_longitude
        self.provider = "UCalgary"

    def __str__(self) -> str:
        return self.__repr__()

    def __repr__(self) -> str:
        return "Observatory(uid=%s, full_name='%s', geodetic_latitude=%s, geodetic_longitude=%s, provider='%s')" % (
            self.uid,
            self.full_name,
            self.geodetic_latitude,
            self.geodetic_longitude,
            self.provider,
        )

    def pretty_print(self):
        """
        A special print output for this class.
        """
        print("Observatory:")
        for var_name in dir(self):
            # exclude methods
            if (var_name.startswith("__") or var_name == "pretty_print"):
                continue

            # convert var to string format we want
            var_value = getattr(self, var_name)
            print("  %-22s: %s" % (var_name, None if var_value is None else var_value))

Methods

def pretty_print(self)

A special print output for this class.

Expand source code

def pretty_print(self):
    """
    A special print output for this class.
    """
    print("Observatory:")
    for var_name in dir(self):
        # exclude methods
        if (var_name.startswith("__") or var_name == "pretty_print"):
            continue

        # convert var to string format we want
        var_value = getattr(self, var_name)
        print("  %-22s: %s" % (var_name, None if var_value is None else var_value))

class Skymap (filename: str, project_uid: str, site_uid: str, imager_uid: str, site_map_latitude: float, site_map_longitude: float, site_map_altitude: float, full_elevation: numpy.ndarray, full_azimuth: numpy.ndarray, full_map_altitude: numpy.ndarray, full_map_latitude: numpy.ndarray, full_map_longitude: numpy.ndarray, generation_info: pyucalgarysrs.data.classes.SkymapGenerationInfo, version: str)

Representation for a skymap file.

Attributes

filename : str

Filename for the skymap file, as an absolute path of its location on the local machine.

project_uid : str

Project unique identifier

site_uid : str

Site unique identifier

imager_uid : str

Imager/device unique identifier

site_map_latitude : float

Geodetic latitude of instrument

site_map_longitude : float

Geodetic longitude of instrument

site_map_altitude : float

Altitude of the instrument (in meters)

full_elevation : ndarray

Elevation angle from horizon, for each image pixel (in degrees)

full_azimuth : ndarray

Local azimuth angle from 0 degrees north, positive moving east (in degrees)

full_map_altitude : ndarray

Altitudes that image coordinates are mapped to (in kilometers)

full_map_latitude : ndarray

Geodetic latitudes of pixel corners, mapped to various altitudes (specified by full_map_altitude)

full_map_longitude : ndarray

Geodetic longitudes of pixel corners, mapped to various altitudes (specified by full_map_altitude)

generation_info : SkymapGenerationInfo

Metadata describing details about this skymap's generation process

version : str

Version of the skymap

dataset : Dataset

The Dataset object for this data.

Expand source code

@dataclass
class Skymap:
    """
    Representation for a skymap file.

    Attributes:
        filename (str): 
            Filename for the skymap file, as an absolute path of its location on the local machine.
        
        project_uid (str): 
            Project unique identifier
        
        site_uid (str): 
            Site unique identifier
        
        imager_uid (str): 
            Imager/device unique identifier
        
        site_map_latitude (float): 
            Geodetic latitude of instrument
        
        site_map_longitude (float): 
            Geodetic longitude of instrument
        
        site_map_altitude (float): 
            Altitude of the instrument (in meters)
        
        full_elevation (ndarray): 
            Elevation angle from horizon, for each image pixel (in degrees)
        
        full_azimuth (ndarray): 
            Local azimuth angle from 0 degrees north, positive moving east (in degrees)
        
        full_map_altitude (ndarray): 
            Altitudes that image coordinates are mapped to (in kilometers)
        
        full_map_latitude (ndarray): 
            Geodetic latitudes of pixel corners, mapped to various altitudes (specified by `full_map_altitude`)
        
        full_map_longitude (ndarray): 
            Geodetic longitudes of pixel corners, mapped to various altitudes (specified by `full_map_altitude`)
        
        generation_info (SkymapGenerationInfo): 
            Metadata describing details about this skymap's generation process
        
        version (str): 
            Version of the skymap
        
        dataset (Dataset): 
            The `Dataset` object for this data.
    """
    filename: str
    project_uid: str
    site_uid: str
    imager_uid: str
    site_map_latitude: float
    site_map_longitude: float
    site_map_altitude: float
    full_elevation: ndarray
    full_azimuth: ndarray
    full_map_altitude: ndarray
    full_map_latitude: ndarray
    full_map_longitude: ndarray
    generation_info: SkymapGenerationInfo
    version: str

    def __str__(self) -> str:
        return self.__repr__()

    def __repr__(self) -> str:
        return "Skymap(project_uid=%s, site_uid=%s, imager_uid=%s, site_map_latitude=%f, site_map_longitude=%f, ...)" % (
            self.project_uid,
            self.site_uid,
            self.imager_uid,
            self.site_map_latitude,
            self.site_map_longitude,
        )

    def pretty_print(self):
        """
        A special print output for this class.
        """
        print("Skymap:")
        for var_name in dir(self):
            # exclude methods
            if (var_name.startswith("__") or var_name == "pretty_print"):
                continue

            # convert var to string format we want
            var_value = getattr(self, var_name)
            var_str = "None"
            if (var_name == "generation_info"):
                var_str = "SkymapGenerationInfo(...)"
            elif (var_value is not None):
                if (isinstance(var_value, ndarray)):
                    var_str = "array(dims=%s, dtype=%s)" % (var_value.shape, var_value.dtype)
                else:
                    var_str = str(var_value)

            # print string for this var
            print("  %-23s: %s" % (var_name, var_str))

    def get_precalculated_altitudes(self):
        """
        Get the altitudes that have been precalculated in this skymap. Units are kilometers.
        """
        alts_km = [float(x / 1000.) for x in self.full_map_altitude]
        return alts_km

Class variables

var filename : str

var full_azimuth : numpy.ndarray

var full_elevation : numpy.ndarray

var full_map_altitude : numpy.ndarray

var full_map_latitude : numpy.ndarray

var full_map_longitude : numpy.ndarray

var generation_info : pyucalgarysrs.data.classes.SkymapGenerationInfo

var imager_uid : str

var project_uid : str

var site_map_altitude : float

var site_map_latitude : float

var site_map_longitude : float

var site_uid : str

var version : str

Methods

def get_precalculated_altitudes(self)

Get the altitudes that have been precalculated in this skymap. Units are kilometers.

Expand source code

def get_precalculated_altitudes(self):
    """
    Get the altitudes that have been precalculated in this skymap. Units are kilometers.
    """
    alts_km = [float(x / 1000.) for x in self.full_map_altitude]
    return alts_km

def pretty_print(self)

A special print output for this class.

Expand source code

def pretty_print(self):
    """
    A special print output for this class.
    """
    print("Skymap:")
    for var_name in dir(self):
        # exclude methods
        if (var_name.startswith("__") or var_name == "pretty_print"):
            continue

        # convert var to string format we want
        var_value = getattr(self, var_name)
        var_str = "None"
        if (var_name == "generation_info"):
            var_str = "SkymapGenerationInfo(...)"
        elif (var_value is not None):
            if (isinstance(var_value, ndarray)):
                var_str = "array(dims=%s, dtype=%s)" % (var_value.shape, var_value.dtype)
            else:
                var_str = str(var_value)

        # print string for this var
        print("  %-23s: %s" % (var_name, var_str))

class UCalgaryManager (aurorax_obj)

The UCalgaryManager object is initialized within every PyAuroraX object. It acts as a way to access the submodules and carry over configuration information in the super class.

Expand source code

class UCalgaryManager:
    """
    The UCalgaryManager object is initialized within every PyAuroraX object. It acts as a way to access 
    the submodules and carry over configuration information in the super class.
    """

    __DEFAULT_DOWNLOAD_N_PARALLEL = 5

    def __init__(self, aurorax_obj):
        self.__aurorax_obj: PyAuroraX = aurorax_obj

        # initialize sub-modules
        self.__readers = ReadManager(self.__aurorax_obj)

    @property
    def readers(self):
        """
        Access to the `read` submodule from within a PyAuroraX object.
        """
        return self.__readers

    def list_datasets(self, name: Optional[str] = None, timeout: Optional[int] = None) -> List[Dataset]:
        """
        List available datasets

        Args:
            name (str): 
                Supply a name used for filtering. If that name is found in the available dataset 
                names received from the API, it will be included in the results. This parameter is
                optional.
            
            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.
            
        Returns:
            A list of [`Dataset`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.Dataset)
            objects.
        
        Raises:
            pyaurorax.exceptions.AuroraXAPIError: An API error was encountered.
        """
        try:
            return self.__aurorax_obj.srs_obj.data.list_datasets(name=name, timeout=timeout)
        except SRSAPIError as e:
            raise AuroraXAPIError(e) from e

    def list_observatories(self,
                           instrument_array: Literal["themis_asi", "rego", "trex_rgb", "trex_nir", "trex_blue"],
                           uid: Optional[str] = None,
                           timeout: Optional[int] = None) -> List[Observatory]:
        """
        List information about observatories

        Args:
            instrument_array (str): 
                The instrument array to list observatories for. Valid values are: themis_asi, rego, 
                trex_rgb, trex_nir, and trex_blue.

            uid (str): 
                Supply a observatory unique identifier used for filtering (usually 4-letter site code). If that UID 
                is found in the available observatories received from the API, it will be included in the results. This 
                parameter is optional.
            
            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.
            
        Returns:
            A list of [`Observatory`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.Observatory)
            objects.
        
        Raises:
            pyaurorax.exceptions.AuroraXAPIError: An API error was encountered.
        """
        try:
            return self.__aurorax_obj.srs_obj.data.list_observatories(instrument_array, uid=uid, timeout=timeout)
        except SRSAPIError as e:
            raise AuroraXAPIError(e) from e

    def list_supported_read_datasets(self) -> List[str]:
        """
        List the datasets which have file reading capabilities supported.

        Returns:
            A list of the dataset names with file reading support.
        """
        return self.__aurorax_obj.srs_obj.data.list_supported_read_datasets()

    def is_read_supported(self, dataset_name: str) -> bool:
        """
        Check if a given dataset has file reading support. 
        
        Not all datasets available in the UCalgary Space Remote Sensing Open Data Platform 
        have special readfile routines in this library. This is because some datasets are 
        in basic formats such as JPG or PNG, so unique functions aren't necessary. We leave 
        it up to the user to open these basic files in whichever way they prefer. Use the 
        `list_supported_read_datasets()` function to see all datasets that have special
        file reading functionality in this library.

        Args:
            dataset_name (str): 
                The dataset name to check if file reading is supported. This parameter 
                is required.
        
        Returns:
            Boolean indicating if file reading is supported.
        """
        return self.__aurorax_obj.srs_obj.data.is_read_supported(dataset_name)

    def download(self,
                 dataset_name: str,
                 start: datetime.datetime,
                 end: datetime.datetime,
                 site_uid: Optional[str] = None,
                 device_uid: Optional[str] = None,
                 n_parallel: int = __DEFAULT_DOWNLOAD_N_PARALLEL,
                 overwrite: bool = False,
                 progress_bar_disable: bool = False,
                 progress_bar_ncols: Optional[int] = None,
                 progress_bar_ascii: Optional[str] = None,
                 progress_bar_desc: Optional[str] = None,
                 timeout: Optional[int] = None) -> FileDownloadResult:
        """
        Download data from the UCalgary Space Remote Sensing Open Data Platform.

        The parameters `dataset_name`, `start`, and `end` are required. All other parameters
        are optional.

        Note that usage of the site and device UID filters applies differently to some datasets.
        For example, both fields can be used for most raw and keogram data, but only site UID can
        be used for skymap datasets, and only device UID can be used for calibration datasets. If 
        fields are specified during a call in which site or device UID is not used, a UserWarning
        is display to provide the user with feedback about this detail.

        Args:
            dataset_name (str): 
                Name of the dataset to download data for. Use the `list_datasets()` function
                to get the possible values for this parameter. One example is "THEMIS_ASI_RAW". 
                Note that dataset names are case sensitive. This parameter is required.

            start (datetime.datetime): 
                Start timestamp to use (inclusive), expected to be in UTC. Any timezone data 
                will be ignored. This parameter is required.

            end (datetime.datetime): 
                End timestamp to use (inclusive), expected to be in UTC. Any timezone data 
                will be ignored. This parameter is required.

            site_uid (str): 
                The site UID to filter for. If specified, data will be downloaded for only the 
                site matching the given value. If excluded, data for all available sites will 
                be downloaded. An example value could be 'atha', meaning all data from the 
                Athabasca observatory will be downloaded for the given dataset name, start, and 
                end times. This parameter is optional.

            device_uid (str): 
                The device UID to filter for. If specified, data will be downloaded for only the
                device matching the given value. If excluded, data for all available devices will
                be downloaded. An example value could be 'themis02', meaning all data matching that
                device will be downloaded for the given dataset name, start, and end times. This
                parameter is optional.

            n_parallel (int): 
                Number of data files to download in parallel. Default value is 5. Adjust as needed 
                for your internet connection. This parameter is optional.

            overwrite (bool): 
                By default, data will not be re-downloaded if it already exists locally. Use 
                the `overwrite` parameter to force re-downloading. Default is `False`. This 
                parameter is optional.

            progress_bar_disable (bool): 
                Disable the progress bar. Default is `False`. This parameter is optional.

            progress_bar_ncols (int): 
                Number of columns for the progress bar (straight passthrough of the `ncols` 
                parameter in a tqdm progress bar). This parameter is optional. See Notes section
                below for further information.
            
            progress_bar_ascii (str): 
                ASCII value to use when constructing the visual aspect of the progress bar (straight 
                passthrough of the `ascii` parameter in a tqdm progress bar). This parameter is 
                optional. See Notes section below for further details.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.

        Returns:
            A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
            object containing details about what data files were downloaded.

        Raises:
            pyaurorax.exceptions.AuroraXDownloadError: an error was encountered while downloading a 
                specific file
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered

        Notes:
        --------
        The `progress_bar_*` parameters can be used to enable/disable/adjust the progress bar. 
        Excluding the `progress_bar_disable` parameter, all others are straight pass-throughs 
        to the tqdm progress bar function. The `progress_bar_ncols` parameter allows for 
        adjusting the width. The `progress_bar_ascii` parameter allows for adjusting the appearance 
        of the progress bar. And the `progress_bar_desc` parameter allows for adjusting the 
        description at the beginning of the progress bar. Further details can be found on the
        [tqdm documentation](https://tqdm.github.io/docs/tqdm/#tqdm-objects).

        Data downloading will use the `download_data_root_path` variable within the super class'
        object ([`PyAuroraX`](../../index.html#pyaurorax.PyAuroraX)) to determine where to save data to. If 
        you'd like to change this path to somewhere else you can change that variable before your
        download() call, like so:

        ```python
        import pyaurorax
        aurorax = pyaurorax.PyAuroraX()
        aurorax.data_download_root_path = "some_new_path"
        aurorax.data.download(dataset_name, start, end)
        ```
        """
        try:
            return self.__aurorax_obj.srs_obj.data.download(
                dataset_name,
                start,
                end,
                site_uid=site_uid,
                device_uid=device_uid,
                n_parallel=n_parallel,
                overwrite=overwrite,
                progress_bar_disable=progress_bar_disable,
                progress_bar_ncols=progress_bar_ncols,
                progress_bar_ascii=progress_bar_ascii,
                progress_bar_desc=progress_bar_desc,
                timeout=timeout,
            )
        except SRSDownloadError as e:
            raise AuroraXDownloadError(e) from e
        except SRSAPIError as e:
            raise AuroraXAPIError(e) from e

    def download_using_urls(self,
                            file_listing_response: FileListingResponse,
                            n_parallel: int = __DEFAULT_DOWNLOAD_N_PARALLEL,
                            overwrite: bool = False,
                            progress_bar_disable: bool = False,
                            progress_bar_ncols: Optional[int] = None,
                            progress_bar_ascii: Optional[str] = None,
                            progress_bar_desc: Optional[str] = None,
                            timeout: Optional[int] = None) -> FileDownloadResult:
        """
        Download data from the UCalgary Space Remote Sensing Open Data Platform using 
        a FileListingResponse object. This would be used in cases where more customization 
        is needed than the generic `download()` function. 
        
        One example of using this function would start by using `get_urls()` to retrieve the
        list of URLs available for download, then further process this list to fewer files
        based on some other requirement (ie. time down-sampling such as one file per hour). 
        Lastly using this function to download the new custom set URLs.

        Args:
            file_listing_response (FileListingResponse): 
                A [`FileListingResponse`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileListingResponse) 
                object returned from a `get_urls()` call, which contains a list of URLs to download 
                for a specific dataset. This parameter is required.

            n_parallel (int): 
                Number of data files to download in parallel. Default value is 5. Adjust as needed 
                for your internet connection. This parameter is optional.

            overwrite (bool): 
                By default, data will not be re-downloaded if it already exists locally. Use 
                the `overwrite` parameter to force re-downloading. Default is `False`. This 
                parameter is optional.

            progress_bar_disable (bool): 
                Disable the progress bar. Default is `False`. This parameter is optional.

            progress_bar_ncols (int): 
                Number of columns for the progress bar (straight passthrough of the `ncols` 
                parameter in a tqdm progress bar). This parameter is optional. See Notes section
                below for further information.
            
            progress_bar_ascii (str): 
                ASCII value to use when constructing the visual aspect of the progress bar (straight 
                passthrough of the `ascii` parameter in a tqdm progress bar). This parameter is 
                optional. See Notes section below for further details.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.

        Returns:
            A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
            object containing details about what data files were downloaded.

        Raises:
            pyaurorax.exceptions.AuroraXDownloadError: an error was encountered while downloading a 
                specific file
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered

        Notes:
        --------
        The `progress_bar_*` parameters can be used to enable/disable/adjust the progress bar. 
        Excluding the `progress_bar_disable` parameter, all others are straight pass-throughs 
        to the tqdm progress bar function. The `progress_bar_ncols` parameter allows for 
        adjusting the width. The `progress_bar_ascii` parameter allows for adjusting the appearance 
        of the progress bar. And the `progress_bar_desc` parameter allows for adjusting the 
        description at the beginning of the progress bar. Further details can be found on the
        [tqdm documentation](https://tqdm.github.io/docs/tqdm/#tqdm-objects).

        Data downloading will use the `download_data_root_path` variable within the super class'
        object ([`PyAuroraX`](../../index.html#pyaurorax.PyAuroraX)) to determine where to save data to. If 
        you'd like to change this path to somewhere else you can change that variable before your
        download() call, like so:

        ```python
        import pyaurorax
        aurorax = pyaurorax.PyAuroraX()
        aurorax.data_download_root_path = "some_new_path"
        aurorax.data.download(dataset_name, start, end)
        ```
        """
        try:
            return self.__aurorax_obj.srs_obj.data.download_using_urls(
                file_listing_response,
                n_parallel=n_parallel,
                overwrite=overwrite,
                progress_bar_disable=progress_bar_disable,
                progress_bar_ncols=progress_bar_ncols,
                progress_bar_ascii=progress_bar_ascii,
                progress_bar_desc=progress_bar_desc,
                timeout=timeout,
            )
        except SRSDownloadError as e:
            raise AuroraXDownloadError(e) from e
        except SRSAPIError as e:
            raise AuroraXAPIError(e) from e

    def get_urls(self,
                 dataset_name: str,
                 start: datetime.datetime,
                 end: datetime.datetime,
                 site_uid: Optional[str] = None,
                 device_uid: Optional[str] = None,
                 timeout: Optional[int] = None) -> FileListingResponse:
        """
        Get URLs of data files

        The parameters `dataset_name`, `start`, and `end` are required. All other parameters
        are optional.

        Note that usage of the site and device UID filters applies differently to some datasets.
        For example, both fields can be used for most raw and keogram data, but only site UID can
        be used for skymap datasets, and only device UID can be used for calibration datasets. If 
        fields are specified during a call in which site or device UID is not used, a UserWarning
        is display to provide the user with feedback about this detail.

        Args:
            dataset_name (str): 
                Name of the dataset to download data for. Use the `list_datasets()` function
                to get the possible values for this parameter. One example is "THEMIS_ASI_RAW". 
                Note that dataset names are case sensitive. This parameter is required.

            start (datetime.datetime): 
                Start timestamp to use (inclusive), expected to be in UTC. Any timezone data 
                will be ignored. This parameter is required.

            end (datetime.datetime): 
                End timestamp to use (inclusive), expected to be in UTC. Any timezone data 
                will be ignored. This parameter is required.

            site_uid (str): 
                The site UID to filter for. If specified, data will be downloaded for only the 
                site matching the given value. If excluded, data for all available sites will 
                be downloaded. An example value could be 'atha', meaning all data from the 
                Athabasca observatory will be downloaded for the given dataset name, start, and 
                end times. This parameter is optional.

            device_uid (str): 
                The device UID to filter for. If specified, data will be downloaded for only the
                device matching the given value. If excluded, data for all available devices will
                be downloaded. An example value could be 'themis02', meaning all data matching that
                device will be downloaded for the given dataset name, start, and end times. This
                parameter is optional.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.
    
        Returns:
            A [`FileListingResponse`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileListingResponse)
            object containing a list of the available URLs, among other values.

        Raises:
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered
        """
        try:
            return self.__aurorax_obj.srs_obj.data.get_urls(
                dataset_name,
                start,
                end,
                site_uid=site_uid,
                device_uid=device_uid,
                timeout=timeout,
            )
        except SRSAPIError as e:
            raise AuroraXAPIError(e) from e

    def read(self,
             dataset: Dataset,
             file_list: Union[List[str], List[Path], str, Path],
             n_parallel: int = 1,
             first_record: bool = False,
             no_metadata: bool = False,
             quiet: bool = False) -> Data:
        """
        Read in data files for a given dataset. Note that only one type of dataset's data
        should be read in using a single call.

        Args:
            dataset (Dataset): 
                The dataset object for which the files are associated with. This parameter is
                required.
            
            file_list (List[str], List[Path], str, Path): 
                The files to read in. Absolute paths are recommended, but not technically
                necessary. This can be a single string for a file, or a list of strings to read
                in multiple files. This parameter is required.

            n_parallel (int): 
                Number of data files to read in parallel using multiprocessing. Default value 
                is 1. Adjust according to your computer's available resources. This parameter 
                is optional.
            
            first_record (bool): 
                Only read in the first record in each file. This is the same as the first_frame
                parameter in the themis-imager-readfile and trex-imager-readfile libraries, and
                is a read optimization if you only need one image per minute, as opposed to the
                full temporal resolution of data (e.g., 3sec cadence). This parameter is optional.
            
            no_metadata (bool): 
                Skip reading of metadata. This is a minor optimization if the metadata is not needed.
                Default is `False`. This parameter is optional.
            
            quiet (bool): 
                Do not print out errors while reading data files, if any are encountered. Any files
                that encounter errors will be, as usual, accessible via the `problematic_files` 
                attribute of the returned `Data` object. This parameter is optional.
        
        Returns:
            A [`Data`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.Data) 
            object containing the data read in, among other values.
        
        Raises:
            pyaurorax.exceptions.AuroraXUnsupportedReadError: an unsupported dataset was used when
                trying to read files.
            pyaurorax.exceptions.AuroraXError: a generic read error was encountered

        Notes:
        ---------
        For users who are familiar with the themis-imager-readfile and trex-imager-readfile
        libraries, the read function provides a near-identical usage. Further improvements have 
        been integrated, and those libraries are anticipated to be deprecated at some point in the
        future.
        """
        # NOTE: we do not wrap the exceptions here, instead we pass the call along
        # to the ReadManager object since the method and exception catching is
        # implemented there. No need to duplicate the exception handling logic.
        return self.__readers.read(
            dataset,
            file_list,
            n_parallel=n_parallel,
            first_record=first_record,
            no_metadata=no_metadata,
            quiet=quiet,
        )

    def download_best_skymap(
        self,
        dataset_name: str,
        site_uid: str,
        timestamp: datetime.datetime,
        timeout: Optional[int] = None,
        overwrite: bool = False,
    ) -> FileDownloadResult:
        """
        Download the skymap file that best matches the parameters supplied.

        Args:
            dataset_name (str): 
                Name of the dataset to download data for. Use the `list_datasets()` function
                to get the possible values for this parameter. One example is "THEMIS_ASI_SKYMAP_IDLSAV". 
                Note that dataset names are case sensitive. This parameter is required.

            site_uid (str): 
                The site UID to evaluate.

            timestamp (datetime.datetime): 
                The timestamp to use for deciding the best skymap, expected to be in UTC. Any timezone 
                data will be ignored. This parameter is required.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.

        Returns:
            A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
            object containing details about what data files were downloaded.

        Raises:
            ValueError: issue with supplied timestamp
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered        
        """
        # get list of all skymap urls for the dataset and site
        start_dt = datetime.datetime(2000, 1, 1)
        end_dt = datetime.datetime.now() + datetime.timedelta(days=5)
        file_listing_obj = self.get_urls(dataset_name, start_dt, end_dt, site_uid=site_uid, timeout=timeout)

        # filter down and find the best skymap for the timestamp supplied
        best_skymap_filename = None
        for url in file_listing_obj.urls:
            # extract start date for this skymap
            url_short = url.replace(file_listing_obj.path_prefix + "/", "")

            # parse filename into several values
            filename_split = os.path.basename(url_short).split('_')
            filename_times_split = filename_split[3].split('-')
            valid_interval_start_dt = datetime.datetime.strptime(filename_times_split[0], "%Y%m%d")

            # check start time
            if (timestamp >= valid_interval_start_dt):
                # valid
                #
                # NOTE: this works because of the order that the list is in already
                best_skymap_filename = url

        # check if we found a skymap
        if (best_skymap_filename is None):
            raise ValueError("Unable to determine a skymap recommendation")

        # set the filename
        file_listing_obj.urls = [best_skymap_filename]
        download_obj = self.download_using_urls(
            file_listing_obj,
            progress_bar_disable=True,
            overwrite=overwrite,
            timeout=timeout,
        )

        # return
        return download_obj

    def download_best_flatfield_calibration(
        self,
        dataset_name: str,
        device_uid: str,
        timestamp: datetime.datetime,
        timeout: Optional[int] = None,
        overwrite: bool = False,
    ) -> FileDownloadResult:
        """
        Download the flatfield calibration file that best matches the parameters supplied.

        Args:
            dataset_name (str): 
                Name of the dataset to download data for. Use the `list_datasets()` function
                to get the possible values for this parameter. One example is "THEMIS_ASI_SKYMAP_IDLSAV". 
                Note that dataset names are case sensitive. This parameter is required.

            device_uid (str): 
                The device UID to evaluate.

            timestamp (datetime.datetime): 
                The timestamp to use for deciding the best skymap, expected to be in UTC. Any timezone 
                data will be ignored. This parameter is required.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.

        Returns:
            A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
            object containing details about what data files were downloaded.

        Raises:
            ValueError: issue with supplied timestamp
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered        
        """
        # get list of all flatfield urls for the dataset and device
        start_dt = datetime.datetime(2000, 1, 1)
        end_dt = datetime.datetime.now() + datetime.timedelta(days=5)
        file_listing_obj = self.get_urls(dataset_name, start_dt, end_dt, device_uid=device_uid, timeout=timeout)

        # filter down and find the best skymap for the timestamp supplied
        best_cal_filename = None
        for url in file_listing_obj.urls:
            # extract start date for this skymap
            url_short = url.replace(file_listing_obj.path_prefix + "/", "")

            # parse filename into several values
            filename_split = os.path.basename(url_short).split('_')
            filename_times_split = filename_split[3].split('-')
            valid_interval_start_dt = datetime.datetime.strptime(filename_times_split[0], "%Y%m%d")

            # check start time
            if (timestamp >= valid_interval_start_dt):
                # valid
                #
                # NOTE: this works because of the order that the list is in already
                best_cal_filename = url

        # check if we found a skymap
        if (best_cal_filename is None):
            raise ValueError("Unable to determine a flatfield calibration recommendation")

        # set the filename
        file_listing_obj.urls = [best_cal_filename]
        download_obj = self.download_using_urls(
            file_listing_obj,
            progress_bar_disable=True,
            overwrite=overwrite,
            timeout=timeout,
        )

        # return
        return download_obj

    def download_best_rayleighs_calibration(
        self,
        dataset_name: str,
        device_uid: str,
        timestamp: datetime.datetime,
        timeout: Optional[int] = None,
        overwrite: bool = False,
    ) -> FileDownloadResult:
        """
        Download the Rayleighs calibration file that best matches the parameters supplied.

        Args:
            dataset_name (str): 
                Name of the dataset to download data for. Use the `list_datasets()` function
                to get the possible values for this parameter. One example is "REGO_CALIBRATION_RAYLEIGHS_IDLSAV". 
                Note that dataset names are case sensitive. This parameter is required.

            device_uid (str): 
                The device UID to evaluate.

            timestamp (datetime.datetime): 
                The timestamp to use for deciding the best calibration file, expected to be in 
                UTC. Any timezone data will be ignored. This parameter is required.

            timeout (int): 
                Represents how many seconds to wait for the API to send data before giving up. The 
                default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
                object. This parameter is optional.

        Returns:
            A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
            object containing details about what data files were downloaded.

        Raises:
            ValueError: issue with supplied timestamp
            pyaurorax.exceptions.AuroraXAPIError: an API error was encountered        
        """
        # get list of all rayleighs urls for the dataset and device
        start_dt = datetime.datetime(2000, 1, 1)
        end_dt = datetime.datetime.now() + datetime.timedelta(days=5)
        file_listing_obj = self.get_urls(dataset_name, start_dt, end_dt, device_uid=device_uid, timeout=timeout)

        # filter down and find the best skymap for the timestamp supplied
        best_cal_filename = None
        for url in file_listing_obj.urls:
            # extract start date for this skymap
            url_short = url.replace(file_listing_obj.path_prefix + "/", "")

            # parse filename into several values
            filename_split = os.path.basename(url_short).split('_')
            filename_times_split = filename_split[3].split('-')
            valid_interval_start_dt = datetime.datetime.strptime(filename_times_split[0], "%Y%m%d")

            # check start time
            if (timestamp >= valid_interval_start_dt):
                # valid
                #
                # NOTE: this works because of the order that the list is in already
                best_cal_filename = url

        # check if we found a skymap
        if (best_cal_filename is None):
            raise ValueError("Unable to determine a Rayleighs calibration recommendation")

        # set the filename
        file_listing_obj.urls = [best_cal_filename]
        download_obj = self.download_using_urls(
            file_listing_obj,
            progress_bar_disable=True,
            overwrite=overwrite,
            timeout=timeout,
        )

        # return
        return download_obj

Instance variables

var readers

Access to the pyaurorax.data.ucalgary.read submodule from within a PyAuroraX object.

Expand source code

@property
def readers(self):
    """
    Access to the `read` submodule from within a PyAuroraX object.
    """
    return self.__readers

Methods

def download(self, dataset_name: str, start: datetime.datetime, end: datetime.datetime, site_uid: Optional[str] = None, device_uid: Optional[str] = None, n_parallel: int = 5, overwrite: bool = False, progress_bar_disable: bool = False, progress_bar_ncols: Optional[int] = None, progress_bar_ascii: Optional[str] = None, progress_bar_desc: Optional[str] = None, timeout: Optional[int] = None) ‑> pyucalgarysrs.data.classes.FileDownloadResult

Download data from the UCalgary Space Remote Sensing Open Data Platform.

The parameters dataset_name, start, and end are required. All other parameters are optional.

Note that usage of the site and device UID filters applies differently to some datasets. For example, both fields can be used for most raw and keogram data, but only site UID can be used for skymap datasets, and only device UID can be used for calibration datasets. If fields are specified during a call in which site or device UID is not used, a UserWarning is display to provide the user with feedback about this detail.

Args

dataset_name : str

Name of the dataset to download data for. Use the list_datasets() function to get the possible values for this parameter. One example is "THEMIS_ASI_RAW". Note that dataset names are case sensitive. This parameter is required.

start : datetime.datetime

Start timestamp to use (inclusive), expected to be in UTC. Any timezone data will be ignored. This parameter is required.

end : datetime.datetime

End timestamp to use (inclusive), expected to be in UTC. Any timezone data will be ignored. This parameter is required.

site_uid : str

The site UID to filter for. If specified, data will be downloaded for only the site matching the given value. If excluded, data for all available sites will be downloaded. An example value could be 'atha', meaning all data from the Athabasca observatory will be downloaded for the given dataset name, start, and end times. This parameter is optional.

device_uid : str

The device UID to filter for. If specified, data will be downloaded for only the device matching the given value. If excluded, data for all available devices will be downloaded. An example value could be 'themis02', meaning all data matching that device will be downloaded for the given dataset name, start, and end times. This parameter is optional.

n_parallel : int

Number of data files to download in parallel. Default value is 5. Adjust as needed for your internet connection. This parameter is optional.

overwrite : bool

By default, data will not be re-downloaded if it already exists locally. Use the overwrite parameter to force re-downloading. Default is False. This parameter is optional.

progress_bar_disable : bool

Disable the progress bar. Default is False. This parameter is optional.

progress_bar_ncols : int

Number of columns for the progress bar (straight passthrough of the ncols parameter in a tqdm progress bar). This parameter is optional. See Notes section below for further information.

progress_bar_ascii : str

ASCII value to use when constructing the visual aspect of the progress bar (straight passthrough of the ascii parameter in a tqdm progress bar). This parameter is optional. See Notes section below for further details.

timeout : int

Represents how many seconds to wait for the API to send data before giving up. The default is 10 seconds, or the api_timeout value in the super class' PyAuroraX object. This parameter is optional.

Returns

A FileDownloadResult object containing details about what data files were downloaded.

Raises

AuroraXDownloadError

an error was encountered while downloading a specific file

AuroraXAPIError

an API error was encountered

Notes:

The progress_bar_* parameters can be used to enable/disable/adjust the progress bar. Excluding the progress_bar_disable parameter, all others are straight pass-throughs to the tqdm progress bar function. The progress_bar_ncols parameter allows for adjusting the width. The progress_bar_ascii parameter allows for adjusting the appearance of the progress bar. And the progress_bar_desc parameter allows for adjusting the description at the beginning of the progress bar. Further details can be found on the tqdm documentation.

Data downloading will use the download_data_root_path variable within the super class' object (PyAuroraX) to determine where to save data to. If you'd like to change this path to somewhere else you can change that variable before your download() call, like so:

import pyaurorax
aurorax = pyaurorax.PyAuroraX()
aurorax.data_download_root_path = "some_new_path"
aurorax.data.download(dataset_name, start, end)

Expand source code

def download(self,
             dataset_name: str,
             start: datetime.datetime,
             end: datetime.datetime,
             site_uid: Optional[str] = None,
             device_uid: Optional[str] = None,
             n_parallel: int = __DEFAULT_DOWNLOAD_N_PARALLEL,
             overwrite: bool = False,
             progress_bar_disable: bool = False,
             progress_bar_ncols: Optional[int] = None,
             progress_bar_ascii: Optional[str] = None,
             progress_bar_desc: Optional[str] = None,
             timeout: Optional[int] = None) -> FileDownloadResult:
    """
    Download data from the UCalgary Space Remote Sensing Open Data Platform.

    The parameters `dataset_name`, `start`, and `end` are required. All other parameters
    are optional.

    Note that usage of the site and device UID filters applies differently to some datasets.
    For example, both fields can be used for most raw and keogram data, but only site UID can
    be used for skymap datasets, and only device UID can be used for calibration datasets. If 
    fields are specified during a call in which site or device UID is not used, a UserWarning
    is display to provide the user with feedback about this detail.

    Args:
        dataset_name (str): 
            Name of the dataset to download data for. Use the `list_datasets()` function
            to get the possible values for this parameter. One example is "THEMIS_ASI_RAW". 
            Note that dataset names are case sensitive. This parameter is required.

        start (datetime.datetime): 
            Start timestamp to use (inclusive), expected to be in UTC. Any timezone data 
            will be ignored. This parameter is required.

        end (datetime.datetime): 
            End timestamp to use (inclusive), expected to be in UTC. Any timezone data 
            will be ignored. This parameter is required.

        site_uid (str): 
            The site UID to filter for. If specified, data will be downloaded for only the 
            site matching the given value. If excluded, data for all available sites will 
            be downloaded. An example value could be 'atha', meaning all data from the 
            Athabasca observatory will be downloaded for the given dataset name, start, and 
            end times. This parameter is optional.

        device_uid (str): 
            The device UID to filter for. If specified, data will be downloaded for only the
            device matching the given value. If excluded, data for all available devices will
            be downloaded. An example value could be 'themis02', meaning all data matching that
            device will be downloaded for the given dataset name, start, and end times. This
            parameter is optional.

        n_parallel (int): 
            Number of data files to download in parallel. Default value is 5. Adjust as needed 
            for your internet connection. This parameter is optional.

        overwrite (bool): 
            By default, data will not be re-downloaded if it already exists locally. Use 
            the `overwrite` parameter to force re-downloading. Default is `False`. This 
            parameter is optional.

        progress_bar_disable (bool): 
            Disable the progress bar. Default is `False`. This parameter is optional.

        progress_bar_ncols (int): 
            Number of columns for the progress bar (straight passthrough of the `ncols` 
            parameter in a tqdm progress bar). This parameter is optional. See Notes section
            below for further information.
        
        progress_bar_ascii (str): 
            ASCII value to use when constructing the visual aspect of the progress bar (straight 
            passthrough of the `ascii` parameter in a tqdm progress bar). This parameter is 
            optional. See Notes section below for further details.

        timeout (int): 
            Represents how many seconds to wait for the API to send data before giving up. The 
            default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
            object. This parameter is optional.

    Returns:
        A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
        object containing details about what data files were downloaded.

    Raises:
        pyaurorax.exceptions.AuroraXDownloadError: an error was encountered while downloading a 
            specific file
        pyaurorax.exceptions.AuroraXAPIError: an API error was encountered

    Notes:
    --------
    The `progress_bar_*` parameters can be used to enable/disable/adjust the progress bar. 
    Excluding the `progress_bar_disable` parameter, all others are straight pass-throughs 
    to the tqdm progress bar function. The `progress_bar_ncols` parameter allows for 
    adjusting the width. The `progress_bar_ascii` parameter allows for adjusting the appearance 
    of the progress bar. And the `progress_bar_desc` parameter allows for adjusting the 
    description at the beginning of the progress bar. Further details can be found on the
    [tqdm documentation](https://tqdm.github.io/docs/tqdm/#tqdm-objects).

    Data downloading will use the `download_data_root_path` variable within the super class'
    object ([`PyAuroraX`](../../index.html#pyaurorax.PyAuroraX)) to determine where to save data to. If 
    you'd like to change this path to somewhere else you can change that variable before your
    download() call, like so:

    ```python
    import pyaurorax
    aurorax = pyaurorax.PyAuroraX()
    aurorax.data_download_root_path = "some_new_path"
    aurorax.data.download(dataset_name, start, end)
    ```
    """
    try:
        return self.__aurorax_obj.srs_obj.data.download(
            dataset_name,
            start,
            end,
            site_uid=site_uid,
            device_uid=device_uid,
            n_parallel=n_parallel,
            overwrite=overwrite,
            progress_bar_disable=progress_bar_disable,
            progress_bar_ncols=progress_bar_ncols,
            progress_bar_ascii=progress_bar_ascii,
            progress_bar_desc=progress_bar_desc,
            timeout=timeout,
        )
    except SRSDownloadError as e:
        raise AuroraXDownloadError(e) from e
    except SRSAPIError as e:
        raise AuroraXAPIError(e) from e

def download_best_flatfield_calibration(self, dataset_name: str, device_uid: str, timestamp: datetime.datetime, timeout: Optional[int] = None, overwrite: bool = False) ‑> pyucalgarysrs.data.classes.FileDownloadResult

Download the flatfield calibration file that best matches the parameters supplied.

Args

dataset_name : str

Name of the dataset to download data for. Use the list_datasets() function to get the possible values for this parameter. One example is "THEMIS_ASI_SKYMAP_IDLSAV". Note that dataset names are case sensitive. This parameter is required.

device_uid : str

The device UID to evaluate.

timestamp : datetime.datetime

The timestamp to use for deciding the best skymap, expected to be in UTC. Any timezone data will be ignored. This parameter is required.

timeout : int

Represents how many seconds to wait for the API to send data before giving up. The default is 10 seconds, or the api_timeout value in the super class' PyAuroraX object. This parameter is optional.

Returns

A FileDownloadResult object containing details about what data files were downloaded.

Raises

ValueError

issue with supplied timestamp

AuroraXAPIError

an API error was encountered

Expand source code

def download_best_flatfield_calibration(
    self,
    dataset_name: str,
    device_uid: str,
    timestamp: datetime.datetime,
    timeout: Optional[int] = None,
    overwrite: bool = False,
) -> FileDownloadResult:
    """
    Download the flatfield calibration file that best matches the parameters supplied.

    Args:
        dataset_name (str): 
            Name of the dataset to download data for. Use the `list_datasets()` function
            to get the possible values for this parameter. One example is "THEMIS_ASI_SKYMAP_IDLSAV". 
            Note that dataset names are case sensitive. This parameter is required.

        device_uid (str): 
            The device UID to evaluate.

        timestamp (datetime.datetime): 
            The timestamp to use for deciding the best skymap, expected to be in UTC. Any timezone 
            data will be ignored. This parameter is required.

        timeout (int): 
            Represents how many seconds to wait for the API to send data before giving up. The 
            default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
            object. This parameter is optional.

    Returns:
        A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
        object containing details about what data files were downloaded.

    Raises:
        ValueError: issue with supplied timestamp
        pyaurorax.exceptions.AuroraXAPIError: an API error was encountered        
    """
    # get list of all flatfield urls for the dataset and device
    start_dt = datetime.datetime(2000, 1, 1)
    end_dt = datetime.datetime.now() + datetime.timedelta(days=5)
    file_listing_obj = self.get_urls(dataset_name, start_dt, end_dt, device_uid=device_uid, timeout=timeout)

    # filter down and find the best skymap for the timestamp supplied
    best_cal_filename = None
    for url in file_listing_obj.urls:
        # extract start date for this skymap
        url_short = url.replace(file_listing_obj.path_prefix + "/", "")

        # parse filename into several values
        filename_split = os.path.basename(url_short).split('_')
        filename_times_split = filename_split[3].split('-')
        valid_interval_start_dt = datetime.datetime.strptime(filename_times_split[0], "%Y%m%d")

        # check start time
        if (timestamp >= valid_interval_start_dt):
            # valid
            #
            # NOTE: this works because of the order that the list is in already
            best_cal_filename = url

    # check if we found a skymap
    if (best_cal_filename is None):
        raise ValueError("Unable to determine a flatfield calibration recommendation")

    # set the filename
    file_listing_obj.urls = [best_cal_filename]
    download_obj = self.download_using_urls(
        file_listing_obj,
        progress_bar_disable=True,
        overwrite=overwrite,
        timeout=timeout,
    )

    # return
    return download_obj

def download_best_rayleighs_calibration(self, dataset_name: str, device_uid: str, timestamp: datetime.datetime, timeout: Optional[int] = None, overwrite: bool = False) ‑> pyucalgarysrs.data.classes.FileDownloadResult

Download the Rayleighs calibration file that best matches the parameters supplied.

Args

dataset_name : str

Name of the dataset to download data for. Use the list_datasets() function to get the possible values for this parameter. One example is "REGO_CALIBRATION_RAYLEIGHS_IDLSAV". Note that dataset names are case sensitive. This parameter is required.

device_uid : str

The device UID to evaluate.

timestamp : datetime.datetime

The timestamp to use for deciding the best calibration file, expected to be in UTC. Any timezone data will be ignored. This parameter is required.

timeout : int

Represents how many seconds to wait for the API to send data before giving up. The default is 10 seconds, or the api_timeout value in the super class' PyAuroraX object. This parameter is optional.

Returns

A FileDownloadResult object containing details about what data files were downloaded.

Raises

ValueError

issue with supplied timestamp

AuroraXAPIError

an API error was encountered

Expand source code

def download_best_rayleighs_calibration(
    self,
    dataset_name: str,
    device_uid: str,
    timestamp: datetime.datetime,
    timeout: Optional[int] = None,
    overwrite: bool = False,
) -> FileDownloadResult:
    """
    Download the Rayleighs calibration file that best matches the parameters supplied.

    Args:
        dataset_name (str): 
            Name of the dataset to download data for. Use the `list_datasets()` function
            to get the possible values for this parameter. One example is "REGO_CALIBRATION_RAYLEIGHS_IDLSAV". 
            Note that dataset names are case sensitive. This parameter is required.

        device_uid (str): 
            The device UID to evaluate.

        timestamp (datetime.datetime): 
            The timestamp to use for deciding the best calibration file, expected to be in 
            UTC. Any timezone data will be ignored. This parameter is required.

        timeout (int): 
            Represents how many seconds to wait for the API to send data before giving up. The 
            default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
            object. This parameter is optional.

    Returns:
        A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
        object containing details about what data files were downloaded.

    Raises:
        ValueError: issue with supplied timestamp
        pyaurorax.exceptions.AuroraXAPIError: an API error was encountered        
    """
    # get list of all rayleighs urls for the dataset and device
    start_dt = datetime.datetime(2000, 1, 1)
    end_dt = datetime.datetime.now() + datetime.timedelta(days=5)
    file_listing_obj = self.get_urls(dataset_name, start_dt, end_dt, device_uid=device_uid, timeout=timeout)

    # filter down and find the best skymap for the timestamp supplied
    best_cal_filename = None
    for url in file_listing_obj.urls:
        # extract start date for this skymap
        url_short = url.replace(file_listing_obj.path_prefix + "/", "")

        # parse filename into several values
        filename_split = os.path.basename(url_short).split('_')
        filename_times_split = filename_split[3].split('-')
        valid_interval_start_dt = datetime.datetime.strptime(filename_times_split[0], "%Y%m%d")

        # check start time
        if (timestamp >= valid_interval_start_dt):
            # valid
            #
            # NOTE: this works because of the order that the list is in already
            best_cal_filename = url

    # check if we found a skymap
    if (best_cal_filename is None):
        raise ValueError("Unable to determine a Rayleighs calibration recommendation")

    # set the filename
    file_listing_obj.urls = [best_cal_filename]
    download_obj = self.download_using_urls(
        file_listing_obj,
        progress_bar_disable=True,
        overwrite=overwrite,
        timeout=timeout,
    )

    # return
    return download_obj

def download_best_skymap(self, dataset_name: str, site_uid: str, timestamp: datetime.datetime, timeout: Optional[int] = None, overwrite: bool = False) ‑> pyucalgarysrs.data.classes.FileDownloadResult

Download the skymap file that best matches the parameters supplied.

Args

dataset_name : str

Name of the dataset to download data for. Use the list_datasets() function to get the possible values for this parameter. One example is "THEMIS_ASI_SKYMAP_IDLSAV". Note that dataset names are case sensitive. This parameter is required.

site_uid : str

The site UID to evaluate.

timestamp : datetime.datetime

The timestamp to use for deciding the best skymap, expected to be in UTC. Any timezone data will be ignored. This parameter is required.

timeout : int

Represents how many seconds to wait for the API to send data before giving up. The default is 10 seconds, or the api_timeout value in the super class' PyAuroraX object. This parameter is optional.

Returns

A FileDownloadResult object containing details about what data files were downloaded.

Raises

ValueError

issue with supplied timestamp

AuroraXAPIError

an API error was encountered

Expand source code

def download_best_skymap(
    self,
    dataset_name: str,
    site_uid: str,
    timestamp: datetime.datetime,
    timeout: Optional[int] = None,
    overwrite: bool = False,
) -> FileDownloadResult:
    """
    Download the skymap file that best matches the parameters supplied.

    Args:
        dataset_name (str): 
            Name of the dataset to download data for. Use the `list_datasets()` function
            to get the possible values for this parameter. One example is "THEMIS_ASI_SKYMAP_IDLSAV". 
            Note that dataset names are case sensitive. This parameter is required.

        site_uid (str): 
            The site UID to evaluate.

        timestamp (datetime.datetime): 
            The timestamp to use for deciding the best skymap, expected to be in UTC. Any timezone 
            data will be ignored. This parameter is required.

        timeout (int): 
            Represents how many seconds to wait for the API to send data before giving up. The 
            default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
            object. This parameter is optional.

    Returns:
        A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
        object containing details about what data files were downloaded.

    Raises:
        ValueError: issue with supplied timestamp
        pyaurorax.exceptions.AuroraXAPIError: an API error was encountered        
    """
    # get list of all skymap urls for the dataset and site
    start_dt = datetime.datetime(2000, 1, 1)
    end_dt = datetime.datetime.now() + datetime.timedelta(days=5)
    file_listing_obj = self.get_urls(dataset_name, start_dt, end_dt, site_uid=site_uid, timeout=timeout)

    # filter down and find the best skymap for the timestamp supplied
    best_skymap_filename = None
    for url in file_listing_obj.urls:
        # extract start date for this skymap
        url_short = url.replace(file_listing_obj.path_prefix + "/", "")

        # parse filename into several values
        filename_split = os.path.basename(url_short).split('_')
        filename_times_split = filename_split[3].split('-')
        valid_interval_start_dt = datetime.datetime.strptime(filename_times_split[0], "%Y%m%d")

        # check start time
        if (timestamp >= valid_interval_start_dt):
            # valid
            #
            # NOTE: this works because of the order that the list is in already
            best_skymap_filename = url

    # check if we found a skymap
    if (best_skymap_filename is None):
        raise ValueError("Unable to determine a skymap recommendation")

    # set the filename
    file_listing_obj.urls = [best_skymap_filename]
    download_obj = self.download_using_urls(
        file_listing_obj,
        progress_bar_disable=True,
        overwrite=overwrite,
        timeout=timeout,
    )

    # return
    return download_obj

def download_using_urls(self, file_listing_response: pyucalgarysrs.data.classes.FileListingResponse, n_parallel: int = 5, overwrite: bool = False, progress_bar_disable: bool = False, progress_bar_ncols: Optional[int] = None, progress_bar_ascii: Optional[str] = None, progress_bar_desc: Optional[str] = None, timeout: Optional[int] = None) ‑> pyucalgarysrs.data.classes.FileDownloadResult

Download data from the UCalgary Space Remote Sensing Open Data Platform using a FileListingResponse object. This would be used in cases where more customization is needed than the generic download() function.

One example of using this function would start by using get_urls() to retrieve the list of URLs available for download, then further process this list to fewer files based on some other requirement (ie. time down-sampling such as one file per hour). Lastly using this function to download the new custom set URLs.

Args

file_listing_response : FileListingResponse

A FileListingResponse object returned from a get_urls() call, which contains a list of URLs to download for a specific dataset. This parameter is required.

n_parallel : int

Number of data files to download in parallel. Default value is 5. Adjust as needed for your internet connection. This parameter is optional.

overwrite : bool

By default, data will not be re-downloaded if it already exists locally. Use the overwrite parameter to force re-downloading. Default is False. This parameter is optional.

progress_bar_disable : bool

Disable the progress bar. Default is False. This parameter is optional.

progress_bar_ncols : int

Number of columns for the progress bar (straight passthrough of the ncols parameter in a tqdm progress bar). This parameter is optional. See Notes section below for further information.

progress_bar_ascii : str

ASCII value to use when constructing the visual aspect of the progress bar (straight passthrough of the ascii parameter in a tqdm progress bar). This parameter is optional. See Notes section below for further details.

timeout : int

Represents how many seconds to wait for the API to send data before giving up. The default is 10 seconds, or the api_timeout value in the super class' PyAuroraX object. This parameter is optional.

Returns

A FileDownloadResult object containing details about what data files were downloaded.

Raises

AuroraXDownloadError

an error was encountered while downloading a specific file

AuroraXAPIError

an API error was encountered

Notes:

The progress_bar_* parameters can be used to enable/disable/adjust the progress bar. Excluding the progress_bar_disable parameter, all others are straight pass-throughs to the tqdm progress bar function. The progress_bar_ncols parameter allows for adjusting the width. The progress_bar_ascii parameter allows for adjusting the appearance of the progress bar. And the progress_bar_desc parameter allows for adjusting the description at the beginning of the progress bar. Further details can be found on the tqdm documentation.

Data downloading will use the download_data_root_path variable within the super class' object (PyAuroraX) to determine where to save data to. If you'd like to change this path to somewhere else you can change that variable before your download() call, like so:

import pyaurorax
aurorax = pyaurorax.PyAuroraX()
aurorax.data_download_root_path = "some_new_path"
aurorax.data.download(dataset_name, start, end)

Expand source code

def download_using_urls(self,
                        file_listing_response: FileListingResponse,
                        n_parallel: int = __DEFAULT_DOWNLOAD_N_PARALLEL,
                        overwrite: bool = False,
                        progress_bar_disable: bool = False,
                        progress_bar_ncols: Optional[int] = None,
                        progress_bar_ascii: Optional[str] = None,
                        progress_bar_desc: Optional[str] = None,
                        timeout: Optional[int] = None) -> FileDownloadResult:
    """
    Download data from the UCalgary Space Remote Sensing Open Data Platform using 
    a FileListingResponse object. This would be used in cases where more customization 
    is needed than the generic `download()` function. 
    
    One example of using this function would start by using `get_urls()` to retrieve the
    list of URLs available for download, then further process this list to fewer files
    based on some other requirement (ie. time down-sampling such as one file per hour). 
    Lastly using this function to download the new custom set URLs.

    Args:
        file_listing_response (FileListingResponse): 
            A [`FileListingResponse`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileListingResponse) 
            object returned from a `get_urls()` call, which contains a list of URLs to download 
            for a specific dataset. This parameter is required.

        n_parallel (int): 
            Number of data files to download in parallel. Default value is 5. Adjust as needed 
            for your internet connection. This parameter is optional.

        overwrite (bool): 
            By default, data will not be re-downloaded if it already exists locally. Use 
            the `overwrite` parameter to force re-downloading. Default is `False`. This 
            parameter is optional.

        progress_bar_disable (bool): 
            Disable the progress bar. Default is `False`. This parameter is optional.

        progress_bar_ncols (int): 
            Number of columns for the progress bar (straight passthrough of the `ncols` 
            parameter in a tqdm progress bar). This parameter is optional. See Notes section
            below for further information.
        
        progress_bar_ascii (str): 
            ASCII value to use when constructing the visual aspect of the progress bar (straight 
            passthrough of the `ascii` parameter in a tqdm progress bar). This parameter is 
            optional. See Notes section below for further details.

        timeout (int): 
            Represents how many seconds to wait for the API to send data before giving up. The 
            default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
            object. This parameter is optional.

    Returns:
        A [`FileDownloadResult`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileDownloadResult) 
        object containing details about what data files were downloaded.

    Raises:
        pyaurorax.exceptions.AuroraXDownloadError: an error was encountered while downloading a 
            specific file
        pyaurorax.exceptions.AuroraXAPIError: an API error was encountered

    Notes:
    --------
    The `progress_bar_*` parameters can be used to enable/disable/adjust the progress bar. 
    Excluding the `progress_bar_disable` parameter, all others are straight pass-throughs 
    to the tqdm progress bar function. The `progress_bar_ncols` parameter allows for 
    adjusting the width. The `progress_bar_ascii` parameter allows for adjusting the appearance 
    of the progress bar. And the `progress_bar_desc` parameter allows for adjusting the 
    description at the beginning of the progress bar. Further details can be found on the
    [tqdm documentation](https://tqdm.github.io/docs/tqdm/#tqdm-objects).

    Data downloading will use the `download_data_root_path` variable within the super class'
    object ([`PyAuroraX`](../../index.html#pyaurorax.PyAuroraX)) to determine where to save data to. If 
    you'd like to change this path to somewhere else you can change that variable before your
    download() call, like so:

    ```python
    import pyaurorax
    aurorax = pyaurorax.PyAuroraX()
    aurorax.data_download_root_path = "some_new_path"
    aurorax.data.download(dataset_name, start, end)
    ```
    """
    try:
        return self.__aurorax_obj.srs_obj.data.download_using_urls(
            file_listing_response,
            n_parallel=n_parallel,
            overwrite=overwrite,
            progress_bar_disable=progress_bar_disable,
            progress_bar_ncols=progress_bar_ncols,
            progress_bar_ascii=progress_bar_ascii,
            progress_bar_desc=progress_bar_desc,
            timeout=timeout,
        )
    except SRSDownloadError as e:
        raise AuroraXDownloadError(e) from e
    except SRSAPIError as e:
        raise AuroraXAPIError(e) from e

def get_urls(self, dataset_name: str, start: datetime.datetime, end: datetime.datetime, site_uid: Optional[str] = None, device_uid: Optional[str] = None, timeout: Optional[int] = None) ‑> pyucalgarysrs.data.classes.FileListingResponse

Get URLs of data files

The parameters dataset_name, start, and end are required. All other parameters are optional.

Note that usage of the site and device UID filters applies differently to some datasets. For example, both fields can be used for most raw and keogram data, but only site UID can be used for skymap datasets, and only device UID can be used for calibration datasets. If fields are specified during a call in which site or device UID is not used, a UserWarning is display to provide the user with feedback about this detail.

Args

dataset_name : str

Name of the dataset to download data for. Use the list_datasets() function to get the possible values for this parameter. One example is "THEMIS_ASI_RAW". Note that dataset names are case sensitive. This parameter is required.

start : datetime.datetime

Start timestamp to use (inclusive), expected to be in UTC. Any timezone data will be ignored. This parameter is required.

end : datetime.datetime

End timestamp to use (inclusive), expected to be in UTC. Any timezone data will be ignored. This parameter is required.

site_uid : str

The site UID to filter for. If specified, data will be downloaded for only the site matching the given value. If excluded, data for all available sites will be downloaded. An example value could be 'atha', meaning all data from the Athabasca observatory will be downloaded for the given dataset name, start, and end times. This parameter is optional.

device_uid : str

The device UID to filter for. If specified, data will be downloaded for only the device matching the given value. If excluded, data for all available devices will be downloaded. An example value could be 'themis02', meaning all data matching that device will be downloaded for the given dataset name, start, and end times. This parameter is optional.

timeout : int

Represents how many seconds to wait for the API to send data before giving up. The default is 10 seconds, or the api_timeout value in the super class' PyAuroraX object. This parameter is optional.

Returns

A FileListingResponse object containing a list of the available URLs, among other values.

Raises

AuroraXAPIError

an API error was encountered

Expand source code

def get_urls(self,
             dataset_name: str,
             start: datetime.datetime,
             end: datetime.datetime,
             site_uid: Optional[str] = None,
             device_uid: Optional[str] = None,
             timeout: Optional[int] = None) -> FileListingResponse:
    """
    Get URLs of data files

    The parameters `dataset_name`, `start`, and `end` are required. All other parameters
    are optional.

    Note that usage of the site and device UID filters applies differently to some datasets.
    For example, both fields can be used for most raw and keogram data, but only site UID can
    be used for skymap datasets, and only device UID can be used for calibration datasets. If 
    fields are specified during a call in which site or device UID is not used, a UserWarning
    is display to provide the user with feedback about this detail.

    Args:
        dataset_name (str): 
            Name of the dataset to download data for. Use the `list_datasets()` function
            to get the possible values for this parameter. One example is "THEMIS_ASI_RAW". 
            Note that dataset names are case sensitive. This parameter is required.

        start (datetime.datetime): 
            Start timestamp to use (inclusive), expected to be in UTC. Any timezone data 
            will be ignored. This parameter is required.

        end (datetime.datetime): 
            End timestamp to use (inclusive), expected to be in UTC. Any timezone data 
            will be ignored. This parameter is required.

        site_uid (str): 
            The site UID to filter for. If specified, data will be downloaded for only the 
            site matching the given value. If excluded, data for all available sites will 
            be downloaded. An example value could be 'atha', meaning all data from the 
            Athabasca observatory will be downloaded for the given dataset name, start, and 
            end times. This parameter is optional.

        device_uid (str): 
            The device UID to filter for. If specified, data will be downloaded for only the
            device matching the given value. If excluded, data for all available devices will
            be downloaded. An example value could be 'themis02', meaning all data matching that
            device will be downloaded for the given dataset name, start, and end times. This
            parameter is optional.

        timeout (int): 
            Represents how many seconds to wait for the API to send data before giving up. The 
            default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
            object. This parameter is optional.

    Returns:
        A [`FileListingResponse`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.FileListingResponse)
        object containing a list of the available URLs, among other values.

    Raises:
        pyaurorax.exceptions.AuroraXAPIError: an API error was encountered
    """
    try:
        return self.__aurorax_obj.srs_obj.data.get_urls(
            dataset_name,
            start,
            end,
            site_uid=site_uid,
            device_uid=device_uid,
            timeout=timeout,
        )
    except SRSAPIError as e:
        raise AuroraXAPIError(e) from e

def is_read_supported(self, dataset_name: str) ‑> bool

Check if a given dataset has file reading support.

Not all datasets available in the UCalgary Space Remote Sensing Open Data Platform have special readfile routines in this library. This is because some datasets are in basic formats such as JPG or PNG, so unique functions aren't necessary. We leave it up to the user to open these basic files in whichever way they prefer. Use the list_supported_read_datasets() function to see all datasets that have special file reading functionality in this library.

Args

dataset_name : str

The dataset name to check if file reading is supported. This parameter is required.

Returns

Boolean indicating if file reading is supported.

Expand source code

def is_read_supported(self, dataset_name: str) -> bool:
    """
    Check if a given dataset has file reading support. 
    
    Not all datasets available in the UCalgary Space Remote Sensing Open Data Platform 
    have special readfile routines in this library. This is because some datasets are 
    in basic formats such as JPG or PNG, so unique functions aren't necessary. We leave 
    it up to the user to open these basic files in whichever way they prefer. Use the 
    `list_supported_read_datasets()` function to see all datasets that have special
    file reading functionality in this library.

    Args:
        dataset_name (str): 
            The dataset name to check if file reading is supported. This parameter 
            is required.
    
    Returns:
        Boolean indicating if file reading is supported.
    """
    return self.__aurorax_obj.srs_obj.data.is_read_supported(dataset_name)

def list_datasets(self, name: Optional[str] = None, timeout: Optional[int] = None) ‑> List[pyucalgarysrs.data.classes.Dataset]

List available datasets

Args

name : str

Supply a name used for filtering. If that name is found in the available dataset names received from the API, it will be included in the results. This parameter is optional.

timeout : int

Represents how many seconds to wait for the API to send data before giving up. The default is 10 seconds, or the api_timeout value in the super class' PyAuroraX object. This parameter is optional.

Returns

A list of Dataset objects.

Raises

AuroraXAPIError

An API error was encountered.

Expand source code

def list_datasets(self, name: Optional[str] = None, timeout: Optional[int] = None) -> List[Dataset]:
    """
    List available datasets

    Args:
        name (str): 
            Supply a name used for filtering. If that name is found in the available dataset 
            names received from the API, it will be included in the results. This parameter is
            optional.
        
        timeout (int): 
            Represents how many seconds to wait for the API to send data before giving up. The 
            default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
            object. This parameter is optional.
        
    Returns:
        A list of [`Dataset`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.Dataset)
        objects.
    
    Raises:
        pyaurorax.exceptions.AuroraXAPIError: An API error was encountered.
    """
    try:
        return self.__aurorax_obj.srs_obj.data.list_datasets(name=name, timeout=timeout)
    except SRSAPIError as e:
        raise AuroraXAPIError(e) from e

def list_observatories(self, instrument_array: Literal['themis_asi', 'rego', 'trex_rgb', 'trex_nir', 'trex_blue'], uid: Optional[str] = None, timeout: Optional[int] = None) ‑> List[pyucalgarysrs.data.classes.Observatory]

List information about observatories

Args

instrument_array : str

The instrument array to list observatories for. Valid values are: themis_asi, rego, trex_rgb, trex_nir, and trex_blue.

uid : str

Supply a observatory unique identifier used for filtering (usually 4-letter site code). If that UID is found in the available observatories received from the API, it will be included in the results. This parameter is optional.

timeout : int

Represents how many seconds to wait for the API to send data before giving up. The default is 10 seconds, or the api_timeout value in the super class' PyAuroraX object. This parameter is optional.

Returns

A list of Observatory objects.

Raises

AuroraXAPIError

An API error was encountered.

Expand source code

def list_observatories(self,
                       instrument_array: Literal["themis_asi", "rego", "trex_rgb", "trex_nir", "trex_blue"],
                       uid: Optional[str] = None,
                       timeout: Optional[int] = None) -> List[Observatory]:
    """
    List information about observatories

    Args:
        instrument_array (str): 
            The instrument array to list observatories for. Valid values are: themis_asi, rego, 
            trex_rgb, trex_nir, and trex_blue.

        uid (str): 
            Supply a observatory unique identifier used for filtering (usually 4-letter site code). If that UID 
            is found in the available observatories received from the API, it will be included in the results. This 
            parameter is optional.
        
        timeout (int): 
            Represents how many seconds to wait for the API to send data before giving up. The 
            default is 10 seconds, or the `api_timeout` value in the super class' `pyaurorax.PyAuroraX`
            object. This parameter is optional.
        
    Returns:
        A list of [`Observatory`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.Observatory)
        objects.
    
    Raises:
        pyaurorax.exceptions.AuroraXAPIError: An API error was encountered.
    """
    try:
        return self.__aurorax_obj.srs_obj.data.list_observatories(instrument_array, uid=uid, timeout=timeout)
    except SRSAPIError as e:
        raise AuroraXAPIError(e) from e

def list_supported_read_datasets(self) ‑> List[str]

List the datasets which have file reading capabilities supported.

Returns

A list of the dataset names with file reading support.

Expand source code

def list_supported_read_datasets(self) -> List[str]:
    """
    List the datasets which have file reading capabilities supported.

    Returns:
        A list of the dataset names with file reading support.
    """
    return self.__aurorax_obj.srs_obj.data.list_supported_read_datasets()

def read(self, dataset: pyucalgarysrs.data.classes.Dataset, file_list: Union[List[str], List[pathlib.Path], str, pathlib.Path], n_parallel: int = 1, first_record: bool = False, no_metadata: bool = False, quiet: bool = False) ‑> pyucalgarysrs.data.classes.Data

Read in data files for a given dataset. Note that only one type of dataset's data should be read in using a single call.

Args

dataset : Dataset

The dataset object for which the files are associated with. This parameter is required.

file_list : List[str], List[Path], str, Path

The files to read in. Absolute paths are recommended, but not technically necessary. This can be a single string for a file, or a list of strings to read in multiple files. This parameter is required.

n_parallel : int

Number of data files to read in parallel using multiprocessing. Default value is 1. Adjust according to your computer's available resources. This parameter is optional.

first_record : bool

Only read in the first record in each file. This is the same as the first_frame parameter in the themis-imager-readfile and trex-imager-readfile libraries, and is a read optimization if you only need one image per minute, as opposed to the full temporal resolution of data (e.g., 3sec cadence). This parameter is optional.

no_metadata : bool

Skip reading of metadata. This is a minor optimization if the metadata is not needed. Default is False. This parameter is optional.

quiet : bool

Do not print out errors while reading data files, if any are encountered. Any files that encounter errors will be, as usual, accessible via the problematic_files attribute of the returned Data object. This parameter is optional.

Returns

A Data object containing the data read in, among other values.

Raises

AuroraXUnsupportedReadError

an unsupported dataset was used when trying to read files.

AuroraXError

a generic read error was encountered

Notes:

For users who are familiar with the themis-imager-readfile and trex-imager-readfile libraries, the read function provides a near-identical usage. Further improvements have been integrated, and those libraries are anticipated to be deprecated at some point in the future.

Expand source code

def read(self,
         dataset: Dataset,
         file_list: Union[List[str], List[Path], str, Path],
         n_parallel: int = 1,
         first_record: bool = False,
         no_metadata: bool = False,
         quiet: bool = False) -> Data:
    """
    Read in data files for a given dataset. Note that only one type of dataset's data
    should be read in using a single call.

    Args:
        dataset (Dataset): 
            The dataset object for which the files are associated with. This parameter is
            required.
        
        file_list (List[str], List[Path], str, Path): 
            The files to read in. Absolute paths are recommended, but not technically
            necessary. This can be a single string for a file, or a list of strings to read
            in multiple files. This parameter is required.

        n_parallel (int): 
            Number of data files to read in parallel using multiprocessing. Default value 
            is 1. Adjust according to your computer's available resources. This parameter 
            is optional.
        
        first_record (bool): 
            Only read in the first record in each file. This is the same as the first_frame
            parameter in the themis-imager-readfile and trex-imager-readfile libraries, and
            is a read optimization if you only need one image per minute, as opposed to the
            full temporal resolution of data (e.g., 3sec cadence). This parameter is optional.
        
        no_metadata (bool): 
            Skip reading of metadata. This is a minor optimization if the metadata is not needed.
            Default is `False`. This parameter is optional.
        
        quiet (bool): 
            Do not print out errors while reading data files, if any are encountered. Any files
            that encounter errors will be, as usual, accessible via the `problematic_files` 
            attribute of the returned `Data` object. This parameter is optional.
    
    Returns:
        A [`Data`](https://docs-pyucalgarysrs.phys.ucalgary.ca/data/classes.html#pyucalgarysrs.data.classes.Data) 
        object containing the data read in, among other values.
    
    Raises:
        pyaurorax.exceptions.AuroraXUnsupportedReadError: an unsupported dataset was used when
            trying to read files.
        pyaurorax.exceptions.AuroraXError: a generic read error was encountered

    Notes:
    ---------
    For users who are familiar with the themis-imager-readfile and trex-imager-readfile
    libraries, the read function provides a near-identical usage. Further improvements have 
    been integrated, and those libraries are anticipated to be deprecated at some point in the
    future.
    """
    # NOTE: we do not wrap the exceptions here, instead we pass the call along
    # to the ReadManager object since the method and exception catching is
    # implemented there. No need to duplicate the exception handling logic.
    return self.__readers.read(
        dataset,
        file_list,
        n_parallel=n_parallel,
        first_record=first_record,
        no_metadata=no_metadata,
        quiet=quiet,
    )