Meteostat 1.5.0 (#53)

* Meteostat 1.5.0

* Linting 1.5.0

Author: clampr

examples/daily/aggregate_regional.py

@@ -8,32 +8,34 @@
 The code is licensed under the MIT license.
 """
 
-from datetime import datetime
-import matplotlib.pyplot as plt
-from meteostat import Stations, Daily
+if __name__ == '__main__':
 
-# Configuration
-Daily.max_threads = 5
+    from datetime import datetime
+    import matplotlib.pyplot as plt
+    from meteostat import Stations, Daily
 
-# Time period
-start = datetime(1980, 1, 1)
-end = datetime(2019, 12, 31)
+    # Configuration
+    Daily.cores = 12
 
-# Get random weather stations in the US
-stations = Stations()
-stations = stations.region('US')
-stations = stations.inventory('daily', (start, end))
-stations = stations.fetch(limit=20, sample=True)
+    # Time period
+    start = datetime(1980, 1, 1)
+    end = datetime(2019, 12, 31)
 
-# Get daily data
-data = Daily(stations, start, end)
+    # Get random weather stations in the US
+    stations = Stations()
+    stations = stations.region('US')
+    stations = stations.inventory('daily', (start, end))
+    stations = stations.fetch(limit=50, sample=True)
 
-# Normalize & aggregate
-data = data.normalize().aggregate('1Y', spatial=True).fetch()
+    # Get daily data
+    data = Daily(stations, start, end)
 
-# Chart title
-TITLE = 'Average US Annual Temperature from 1980 to 2019'
+    # Normalize & aggregate
+    data = data.normalize().aggregate('1Y', spatial=True).fetch()
 
-# Plot chart
-data.plot(y=['tavg'], title=TITLE)
-plt.show()
+    # Chart title
+    TITLE = 'Average US Annual Temperature from 1980 to 2019'
+
+    # Plot chart
+    data.plot(y=['tavg'], title=TITLE)
+    plt.show()

examples/hourly/performance.py

@@ -0,0 +1,34 @@
+"""
+Example: Hourly point data performance
+
+Meteorological data provided by Meteostat (https://dev.meteostat.net)
+under the terms of the Creative Commons Attribution-NonCommercial
+4.0 International Public License.
+
+The code is licensed under the MIT license.
+"""
+
+if __name__ == '__main__':
+
+    from timeit import default_timer as timer
+
+    # Get start time
+    s = timer()
+
+    # Run script
+    from datetime import datetime
+    from meteostat import Hourly
+
+    Hourly.cores = 12
+
+    start = datetime(1960, 1, 1)
+    end = datetime(2021, 1, 1, 23, 59)
+
+    data = Hourly('10637', start, end, timezone='Europe/Berlin')
+    data = data.fetch()
+
+    # Get end time
+    e = timer()
+
+    # Print performance
+    print(e - s)

meteostat/__init__.py

@@ -12,8 +12,10 @@
 """
 
 __appname__ = 'meteostat'
-__version__ = '1.4.6'
+__version__ = '1.5.0'
 
+from .interface.base import Base
+from .interface.timeseries import Timeseries
 from .interface.stations import Stations
 from .interface.point import Point
 from .interface.hourly import Hourly

meteostat/core/cache.py

@@ -42,7 +42,10 @@ def file_in_cache(
 
     # Make sure the cache directory exists
     if not os.path.exists(directory):
-        os.makedirs(directory)
+        try:
+            os.makedirs(directory)
+        except FileExistsError:
+            pass
 
     # Return the file path if it exists
     if os.path.isfile(path) and time.time() - \

meteostat/core/config.py

@@ -9,6 +9,7 @@
 """
 
 from urllib.error import HTTPError
+from multiprocessing import Pool
 from multiprocessing.pool import ThreadPool
 from typing import Callable
 import pandas as pd
@@ -18,27 +19,49 @@
 def processing_handler(
     datasets: list,
     load: Callable[[dict], None],
-    max_threads: int
+    cores: int,
+    threads: int
 ) -> None:
     """
-    Load multiple datasets simultaneously
+    Load multiple datasets (simultaneously)
     """
 
-    # Single-thread processing
-    if max_threads < 2:
+    # Data output
+    output = []
 
-        for dataset in datasets:
-            load(*dataset)
+    # Multi-core processing
+    if cores > 1 and len(datasets) > 1:
+
+        # Create process pool
+        with Pool(cores) as pool:
+
+            # Process datasets in pool
+            output = pool.starmap(load, datasets)
+
+            # Wait for Pool to finish
+            pool.close()
+            pool.join()
 
     # Multi-thread processing
+    elif threads > 1 and len(datasets) > 1:
+
+        # Create process pool
+        with ThreadPool(cores) as pool:
+
+            # Process datasets in pool
+            output = pool.starmap(load, datasets)
+
+            # Wait for Pool to finish
+            pool.close()
+            pool.join()
+
+    # Single-thread processing
     else:
 
-        pool = ThreadPool(max_threads)
-        pool.starmap(load, datasets)
+        for dataset in datasets:
+            output.append(load(*dataset))
 
-        # Wait for Pool to finish
-        pool.close()
-        pool.join()
+    return pd.concat(output)
 
 
 def load_handler(

meteostat/core/loader.py

@@ -8,12 +8,27 @@
 The code is licensed under the MIT license.
 """
 
+import os
+
 
 class Base:
 
     """
     Base class that provides features which are used across the package
     """
 
-    # Import configuration
-    from meteostat.core.config import endpoint, cache_dir, max_age, max_threads
+    # Base URL of the Meteostat bulk data interface
+    endpoint: str = 'https://bulk.meteostat.net/v2/'
+
+    # Location of the cache directory
+    cache_dir: str = os.path.expanduser(
+        '~') + os.sep + '.meteostat' + os.sep + 'cache'
+
+    # Maximum age of a cached file in seconds
+    max_age: int = 24 * 60 * 60
+
+    # Number of cores used for processing files
+    cores: int = 1
+
+    # Number of threads used for processing files
+    threads: int = 1

meteostat/interface/base.py

@@ -16,11 +16,11 @@
 from meteostat.core.loader import processing_handler, load_handler
 from meteostat.utilities.validations import validate_series
 from meteostat.utilities.aggregations import degree_mean, weighted_average
-from meteostat.interface.base import Base
+from meteostat.interface.timeseries import Timeseries
 from meteostat.interface.point import Point
 
 
-class Daily(Base):
+class Daily(Timeseries):
 
     """
     Retrieve daily weather observations for one or multiple weather stations or
@@ -30,21 +30,6 @@ class Daily(Base):
     # The cache subdirectory
     cache_subdir: str = 'daily'
 
-    # The list of weather Stations
-    _stations: pd.Index = None
-
-    # The start date
-    _start: datetime = None
-
-    # The end date
-    _end: datetime = None
-
-    # Include model data?
-    _model: bool = True
-
-    # The data frame
-    _data: pd.DataFrame = pd.DataFrame()
-
     # Default frequency
     _freq: str = '1D'
 
@@ -143,14 +128,11 @@ def _load(
             # Get time index
             time = df.index.get_level_values('time')
 
-            # Filter & append
-            self._data = self._data.append(
-                df.loc[(time >= self._start) & (time <= self._end)])
+            # Filter & return
+            return df.loc[(time >= self._start) & (time <= self._end)]
 
-        else:
-
-            # Append
-            self._data = self._data.append(df)
+        # Return
+        return df
 
     def _get_data(self) -> None:
         """
@@ -168,12 +150,11 @@ def _get_data(self) -> None:
                 ))
 
             # Data Processing
-            processing_handler(datasets, self._load, self.max_threads)
+            return processing_handler(
+                datasets, self._load, self.cores, self.threads)
 
-        else:
-
-            # Empty DataFrame
-            self._data = pd.DataFrame(columns=[*self._types])
+        # Empty DataFrame
+        return pd.DataFrame(columns=[*self._types])
 
     def _resolve_point(
         self,
@@ -263,7 +244,7 @@ def __init__(
         self._model = model
 
         # Get data for all weather stations
-        self._get_data()
+        self._data = self._get_data()
 
         # Interpolate data
         if isinstance(loc, Point):
@@ -279,14 +260,3 @@ def expected_rows(self) -> int:
         """
 
         return (self._end - self._start).days + 1
-
-    # Import methods
-    from meteostat.series.normalize import normalize
-    from meteostat.series.interpolate import interpolate
-    from meteostat.series.aggregate import aggregate
-    from meteostat.series.convert import convert
-    from meteostat.series.coverage import coverage
-    from meteostat.series.count import count
-    from meteostat.series.fetch import fetch
-    from meteostat.series.stations import stations
-    from meteostat.core.cache import clear_cache