Commit 2ae7da2e authored by Hugo Cruz's avatar Hugo Cruz
Browse files

Auto-saving for huggcruzz@gmail.com on branch wip-envass from commit d92061e6

parent d92061e6
Pipeline #329894 passed with stage
in 20 seconds
%% Cell type:code id: tags:
``` python
import os
import yaml
import netCDF4
import numpy as np
import xarray as xr
import pandas as pd
from datetime import datetime, timezone
import glob
with open("scripts/input_python.yaml", "r") as f:
directories = yaml.load(f, Loader=yaml.FullLoader)
files = glob.glob(directories["Level1B_dir"]+"*.nc")
files.sort()
dataset = xr.open_mfdataset(files, decode_times = False)
```
%% Cell type:code id: tags:
``` python
def advanced_quality_flags(df, json_path="quality_assurance.json"):
"""
input :
- df is a dataframe of level 1B where basic check have been performed
- json path: path for the advanced quality check json file, produced by the jupyter notebook
output:
- dictionnary where the dataframe is stored with updated advanced quality checks
"""
quality_assurance_dict = json.load(open(json_path))
var_name = quality_assurance_dict.keys()
advanced_df = df.copy()
for var in var_name:
if quality_assurance_dict[var]:
qa = qualityassurance(np.array(df[var]), np.array(df["time"]), **quality_assurance_dict[var]["advanced"])
advanced_df[var + "_qual"].values[np.array(qa, dtype=bool)] = 1
return advanced_df
```
%% Cell type:code id: tags:
``` python
import json
from envass import qualityassurance
advanced_dataset = advanced_quality_flags(dataset, json_path="quality_assurance.json")
dataset.close()
for file in files:
dset = netCDF4.Dataset(file, 'r+')
idx = np.where((advanced_dataset["time"] >= dset["time"][0]) & (advanced_dataset["time"] <= dset["time"][-1]))[0]
for var in dset.variables:
if "_qual" in var:
if advanced_dataset[var][:].ndim==1:
dset[var][:] = np.array(advanced_dataset[var][:][idx], dtype=bool)
else:
if advanced_dataset[var][:].ndim==2:
dset[var][:] = np.array(advanced_dataset[var][:][:,idx], dtype=bool)
else:
print("didn't recognized number of dimensions")
dset.close()
```
%%%% Output: stream
2D array recognized
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
```
This source diff could not be displayed because it is stored in LFS. You can view the blob instead.
This source diff could not be displayed because it is stored in LFS. You can view the blob instead.
This source diff could not be displayed because it is stored in LFS. You can view the blob instead.
......@@ -4,3 +4,17 @@
09:52:25.393481 Reading Level 1 data
09:52:25.427389 Apply advance quality checks to data
09:52:25.588053 Update NetCDF files with new QA
15:31:40.256628 Performing advanced quality check
15:31:40.258913 Reading Level 1 data
15:31:40.282969 Apply advance quality checks to data
15:31:40.752661 Update NetCDF files with new QA
15:32:21.991738 Performing advanced quality check
15:32:21.993817 Reading Level 1 data
15:32:22.018143 Apply advance quality checks to data
15:32:22.533482 Update NetCDF files with new QA
15:39:04.029129 Performing advanced quality check
15:39:04.031568 Reading Level 1 data
15:39:35.011273 Performing advanced quality check
15:39:35.013755 Reading Level 1 data
15:39:35.048727 Apply advance quality checks to data
15:39:35.425286 Update NetCDF files with new QA
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment