Large refactoring of data management

conda/meta.yaml

@@ -10,7 +10,7 @@ source:
 build:
   noarch: python
   preserve_egg_dir: True
-  number: 3
+  number: 0
   script: {{PYTHON}} setup.py install 
 
 requirements:

src/openalea/strawberry/application/misc.py

@@ -3,18 +3,18 @@
 from io import StringIO
 import pandas as pd
 import base64
+
 from ipywidgets import HTML
 import ipywidgets as widgets
+
 from oawidgets.plantgl import PlantGL
 
 import openalea.strawberry
 from openalea.mtg.io import write_mtg
 from openalea.mtg import MTG
-from openalea.strawberry.application.layout import layout_dataframe, layout_output_wgt, layout_visu3d
-from openalea.deploy.shared_data import shared_data
 
-
-data_directory = shared_data(openalea.strawberry.__path__)
+from openalea.strawberry.data import data_directory
+from openalea.strawberry.application.layout import layout_dataframe, layout_output_wgt, layout_visu3d
 
 
 if layout_dataframe == "qgrid":

src/openalea/strawberry/data.py

@@ -1,7 +1,9 @@
 """ Module providing shared data access
 """
+from pathlib import Path
 
 from openalea.deploy.shared_data import shared_data
+import openalea.strawberry
 
 
 data_directory = shared_data(openalea.strawberry.__path__)

src/openalea/strawberry/import_mtgfile.py

@@ -4,14 +4,15 @@
 import pandas as pd
 import numpy as np
 from openpyxl import load_workbook
+
 from openalea.plantgl.all import*
 from openalea.mtg import io
 from openalea.mtg import algo
 from openalea.lpy import *
 
 import openalea.strawberry
 from openalea.mtg import MTG, algo
-from openalea.deploy.shared_data import shared_data
+from openalea.strawberry.data import data_directory
 
 
 def name(f):
@@ -34,7 +35,7 @@ def import_mtgfile(filename):
     :rtype: MTG
     """    
     filenames = filename
-    files = shared_data(openalea.strawberry).glob('*.mtg')
+    files = data_directory.glob('*.mtg')
     mtg_path = dict((name(f), f) for f in files)
     mtgfile = dict((k,f) for k,f in mtg_path.items() if k in filenames)
     if len(filenames) == 1:

test/test_analysis.py

@@ -1,15 +1,18 @@
+import pandas as pd
+from openalea.mtg.io import read_mtg_file
+
 from openalea.strawberry.analysis import extract_at_module_scale, extract_at_node_scale
 from openalea.strawberry.analysis import occurence_module_order_along_time, prob_axillary_production
-from openalea.deploy.shared_data import shared_data
 import openalea.strawberry
-from openalea.mtg.io import read_mtg_file
+from openalea.strawberry.data import data_directory
+
 
 def name(f):
     return f.basename().splitext()[0]
 
 
 def test_extract_at_module_scale():
-    files = shared_data(openalea.strawberry).glob('*.mtg')
+    files = data_directory.glob('*.mtg')
     mtg_path = dict((name(f), f) for f in files)
     gariguette = read_mtg_file(mtg_path['Gariguette'])
     gariguette_extraction_at_module_scale = extract_at_module_scale(gariguette)
@@ -18,13 +21,19 @@ def test_extract_at_module_scale():
     gariguette_frequency = occurence_module_order_along_time(data= gariguette_extraction_at_module_scale,frequency_type= "cdf")
     assert len(gariguette_frequency) == 6
 
-    mean= gariguette_extraction_at_module_scale.groupby(["Genotype", "order"]).mean()
-    sd= gariguette_extraction_at_module_scale.groupby(["Genotype", "order"]).std()
+
+    # remove object value from mean & std
+    fd = gariguette_extraction_at_module_scale
+    fd.date = pd.to_datetime(fd.date)
+    props = [x for x in fd if x in ["Genotype", "order"] or fd[x].dtype!=object]
+
+    mean= fd.filter(props).groupby(["Genotype", "order"]).mean()
+    sd= fd.filter(props).groupby(["Genotype", "order"]).std()
     assert len(mean) == 6
     assert len(sd) == 6
 
 def test_extraction_at_node_scale():
-    files = shared_data(openalea.strawberry).glob('*.mtg')
+    files = data_directory.glob('*.mtg')
     mtg_path = dict((name(f), f) for f in files)
     gariguette = read_mtg_file(mtg_path['Gariguette'])
     gariguette_extraction_at_node_scale = extract_at_node_scale(gariguette)

test/test_extraction.py

@@ -1,17 +1,17 @@
-from openalea.deploy.shared_data import shared_data
 import openalea.strawberry
 from openalea.mtg.io import read_mtg_file
 from openalea.mtg.algo import union
 from openalea.mtg import MTG
 
 
 from openalea.strawberry.analysis import extract_at_module_scale, extract_at_node_scale, extract_at_plant_scale
+from openalea.strawberry.data import data_directory
 
 def name(f):
     return f.basename().splitext()[0]
 
 def test_extract_at_module_scale():
-    files = shared_data(openalea.strawberry).glob('*.mtg')
+    files = data_directory.glob('*.mtg')
 
     mtg_path = dict((name(f), f) for f in files)
     mtg = MTG()
@@ -32,7 +32,7 @@ def test_extract_at_module_scale():
 
 
 def test_extract_at_node_scale():
-    files = shared_data(openalea.strawberry).glob('*.mtg')
+    files = data_directory.glob('*.mtg')
 
     mtg_path = dict((name(f), f) for f in files)
     mtg = MTG()
@@ -53,7 +53,7 @@ def test_extract_at_node_scale():
 
 
 def test_extract_at_plant_scale():
-    files = shared_data(openalea.strawberry).glob('*.mtg')
+    files = data_directory.glob('*.mtg')
 
     mtg_path = dict((name(f), f) for f in files)
     mtg = MTG()

test/test_import_data.py

@@ -1,23 +1,23 @@
-from openalea.deploy.shared_data import shared_data
+import os
 
 import openalea.strawberry
 from openalea.strawberry.import_mtgfile import import_mtgfile, import_mtg_from_csv, strawberry_reader_csv
+from openalea.strawberry.data import data_directory
 
 
-
+data_dir = data_directory
 
 def name(f):
     return f.basename().splitext()[0]
 
-mtg_files = shared_data(openalea.strawberry).glob('*.mtg')
+mtg_files = data_dir.glob('*.mtg')
 
-excel_files = shared_data(openalea.strawberry).glob('*.xlsx')
-import os
+excel_files = data_dir.glob('*.xlsx')
 
 def test_import_mtgfile():
     """test import_mtgfile function by filename or list of filename
     """
-    mtg_files = shared_data(openalea.strawberry).glob('*.mtg')
+    mtg_files = data_dir.glob('*.mtg')
     mtg_path= dict((name(f), f) for f in mtg_files) 
     genotypes= list(mtg_path.keys())
 
@@ -31,14 +31,14 @@ def test_import_mtgfile():
 def test_import_mtg_from_csv():
     """test import of mtg from excel files
     """
-    excel_files = shared_data(openalea.strawberry).glob('*/*.xlsx')
+    excel_files = data_dir.glob('*/*.xlsx')
     g= import_mtg_from_csv(files=excel_files,first_property="experimental_names",symbol_at_scale=dict(P=1,T=2, F=3, f=3, b=3, HT=3, bt=3, ht=3,s=3)) 
     assert (isinstance(g, openalea.mtg.mtg.MTG),"data not exist or data not respect the formalism")    
 
 def test_strawberry_reader_csv():
     """ test one excel files
     """
-    excel_files = shared_data(openalea.strawberry).glob('*/*.xlsx')
+    excel_files = data_dir.glob('*/*.xlsx')
     for file in excel_files:
         g=strawberry_reader_csv(file=file)
         assert (isinstance(g, openalea.mtg.mtg.MTG), "data not exist or data in"+ file + "not respect the formalism")   

test/test_visualization.py

@@ -1,25 +1,26 @@
-from openalea.deploy.shared_data import shared_data
-import openalea.strawberry
+import openalea
 from openalea.mtg.io import read_mtg_file
 from openalea.mtg.algo import orders, split
-import openalea
+
 from oawidgets.plantgl import PlantGL
+import openalea.strawberry
 from openalea.strawberry import visu2d, visu3d
+from openalea.strawberry.data import data_directory
 
 
 def name(f):
     return f.basename().splitext()[0]
 
 def test_import_mtg():
-    files = shared_data(openalea.strawberry).glob('*.mtg')
+    files = data_directory.glob('*.mtg')
     mtg_path = dict((name(f), f) for f in files)
     gariguette = read_mtg_file(mtg_path['Gariguette'])
     straws = split(gariguette)
     assert isinstance(straws[0], openalea.mtg.mtg.MTG)
 
 
 def test_3D():
-    files = shared_data(openalea.strawberry).glob('*.mtg')
+    files = data_directory.glob('*.mtg')
     mtg_path = dict((name(f), f) for f in files)
     gariguette = read_mtg_file(mtg_path['Gariguette'])
     gariguette.properties()['order'] = orders(gariguette)
@@ -34,7 +35,7 @@ def test_3D():
 
 
 def __test_2D():
-    files = shared_data(openalea.strawberry).glob('*.mtg')
+    files = data_directory.glob('*.mtg')
     mtg_path = dict((name(f), f) for f in files)
     gariguette = read_mtg_file(mtg_path['Gariguette'])
     gariguette.properties()['order'] = orders(gariguette)

test/test_waffle.py

@@ -1,17 +1,18 @@
-from pathlib import Path
 import os
+from pathlib import Path
 from openalea.mtg.io import read_mtg_file, write_mtg
-from openalea.strawberry.analysis import extract_at_node_scale, extract_at_module_scale
-from openalea.deploy.shared_data import shared_data
-import openalea.strawberry
 
+import openalea.strawberry
+from openalea.strawberry.analysis import extract_at_node_scale, extract_at_module_scale
 from openalea.strawberry.analysis import df2waffle
+from openalea.strawberry.data import data_directory
+
 
 def name(f):
     return f.basename().splitext()[0]
 
 def test_df2waffle():
-    files = shared_data(openalea.strawberry).glob('*.mtg')
+    files = data_directory.glob('*.mtg')
     mtg_path = dict((name(f), f) for f in files)
     mtg = read_mtg_file(mtg_path['Capriss'])