In [1]:
from specio import specread
from os import path
import glob
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
In [2]:
from SPARQLWrapper import SPARQLWrapper, JSON
import re

sparql = SPARQLWrapper("https://opendata1.opendata.u-psud.fr/sparql/")
sparql.setQuery("""

prefix projectInvivo: <http://invivo.lipsys2.u-psud.fr/wiki/Data:Analyse_des_macrophages#>
prefix daapp: <http://daap.eu/wiki/Data:Project#> 
prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
prefix vcard: <http://www.w3.org/2006/vcard/ns#>

select DISTINCT  ?picture
where {
  <http://daap.eu/wiki/Analyse_des_macrophages_M├ęthode_2_Campagne_2_:_2017> 
  projectInvivo:hasSlide  ?slide .
    ?slide projectInvivo:hasPicture ?picture .
} 

""")
sparql.setReturnFormat(JSON)
results = sparql.query().convert()

filesNAS = [result["picture"]["value"]  for result in results["results"]["bindings"]]
filenames = sorted([re.sub('file://filer.ups.u-psud.fr/archivagesciences\$/Chimie%20Analytique/projets/Macrophages_imagerie',
                    '/home/jupyter/Notebooks/htdocs/Macrophages_imagerie', file) for file in filesNAS])
print("\n".join(filenames))
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170418STD_L01.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170418STD_L03.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170419EPA_L03.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170419STD_L02.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170420EDL_L01.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170420EPA_L01.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170420LDL_L01.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170421EDL_L02.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170421EPA_L02.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170421LDL_L02.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170422LDL_L03.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170424EDL_L03.fsm
In [3]:
# OLD par fichier
# DATA_PATH = ("/home/jupyter/Notebooks/Macrophages_imagerie/Methode_02/Campagne_02")
# EXT_FILE = "*.fsm"
# #filenames = sorted(glob.glob(path.join(DATA_PATH, EXT_FILE)))
# filenames = glob.glob(path.join(DATA_PATH, EXT_FILE))
In [4]:
print("\n".join(filenames))
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170418STD_L01.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170418STD_L03.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170419EPA_L03.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170419STD_L02.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170420EDL_L01.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170420EPA_L01.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170420LDL_L01.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170421EDL_L02.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170421EPA_L02.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170421LDL_L02.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170422LDL_L03.fsm
/home/jupyter/Notebooks/htdocs/Macrophages_imagerie/Methode_02/Campagne_02/20170424EDL_L03.fsm
In [5]:
# "/home/jupyter/Notebooks/Macrophages_imagerie/Methode_02/Campagne_02/20170418STD_L01.fsm" == "/home/jupyter/Notebooks/Macrophages_imagerie/Methode_02/Campagne_02/20170418STD_L01.fsm"
In [6]:
# i = 0
# fsort1 = sorted(filenames)
# fsort2 = sorted(filenames2)

# for f in fsort1:    
#     print(fsort1[i])
#     print(fsort2[i])
#     print(fsort1[i] == fsort2[i])
#     i = i + 1
In [7]:
list_spec = {}
for f in filenames:
    list_spec[f] = specread(f)
In [8]:
print(len(list_spec))
12
In [9]:
cell_medium = ["EDL", "LDL", "STD", "EPA"]
dict_ir = {k: [] for k in cell_medium}
print(dict_ir)
{'EDL': [], 'LDL': [], 'STD': [], 'EPA': []}
In [10]:
#KR
for key in dict_ir.keys():
    for f in filenames:
        if key in f:
             dict_ir[key].append(list_spec[f])
In [11]:
#OLD VERSION

# for key in dict_ir.keys():
#     for f in filenames:
#         if key in f:
#             dict_ir[key].append(specread(f))
In [12]:
print(dict_ir)
{'EDL': [Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[102.41041  102.50108  102.6334   ... 117.68095  124.63407  126.097404]
 [102.1833   102.29499  102.5289   ... 129.85854  138.07028  127.65028 ]
 [102.0728   102.15745  102.36862  ...  97.370964  96.82941   95.83801 ]
 ...
 [101.20574  101.14632  101.353165 ...  96.523544 110.76741  124.48135 ]
 [102.002625 102.18558  102.52742  ... 107.94787  121.876335 141.33583 ]
 [102.98535  103.009186 103.113174 ...  98.65173  115.48307  134.8786  ]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': '20170420EDL_L01', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': 1109.5000233451517, 'y_init': -137.49998758236615, 'z_init': 4000.0, 'n_x': 189, 'n_y': 101, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Thu Apr 20 08:46:14 2017', 'image_name': 'Default', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170420EDL_L01.fsm'} 
, Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[ 99.4711    99.47538   99.705635 ...  80.75171   87.11026   91.1909  ]
 [101.53501  101.57827  101.71788  ... 128.05081  143.01501  141.09897 ]
 [104.15017  103.99866  104.21387  ... 115.34351  109.42525  102.67608 ]
 ...
 [  0.         0.         0.       ...   0.         0.         0.      ]
 [  0.         0.         0.       ...   0.         0.         0.      ]
 [  0.         0.         0.       ...   0.         0.         0.      ]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': '20170421EDL_L02', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': 1308.2500186264508, 'y_init': 58.25001167257584, 'z_init': 4000.0, 'n_x': 151, 'n_y': 95, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Fri Apr 21 13:01:10 2017', 'image_name': 'Default', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170421EDL_L02.fsm'} 
, Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[ 99.18622   99.17415   98.970215 ...  89.40593   98.58076  100.90654 ]
 [ 99.61621   99.58943   99.66678  ... 100.6327    92.2831    86.76073 ]
 [100.368095 100.36098  100.24655  ... 108.87523  107.81979  100.7395  ]
 ...
 [102.59632  102.52495  102.4322   ... 109.580574  95.95844   89.17854 ]
 [102.71568  102.57474  102.39938  ... 134.68396  140.88217  134.755   ]
 [102.33903  102.52533  102.37818  ... 102.41948  110.08465  114.507416]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': '20170424EDL_L03', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': 4327.750023096799, 'y_init': 3063.6250152736893, 'z_init': 4000.0, 'n_x': 187, 'n_y': 124, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Mon Apr 24 10:21:44 2017', 'image_name': 'Default', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170424EDL_L03.fsm'} 
], 'LDL': [Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[ 96.76078   96.78547   96.85925  ...  89.256004  85.60106   99.70127 ]
 [ 96.76285   96.898605  97.187126 ...  92.758514 110.21162  122.25818 ]
 [ 97.85429   97.894966  98.22846  ...  99.75266  105.9302   110.99608 ]
 ...
 [ 94.66745   94.7308    94.83478  ...  90.19859   98.231445 122.904625]
 [ 93.45002   93.701256  93.778564 ...  88.477104  83.776886  94.74307 ]
 [ 90.18729   90.25069   90.468185 ...  97.729935 112.131355 126.26607 ]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': '20170420LDL_L01', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': 2738.625020240743, 'y_init': 4659.625017260511, 'z_init': 4000.0, 'n_x': 164, 'n_y': 140, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Thu Apr 20 12:54:10 2017', 'image_name': 'Default', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170420LDL_L01.fsm'} 
, Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[ 91.264046    91.165306    91.232056   ...  98.369286   112.256485
  125.01442   ]
 [ 92.64506     92.759186    92.896645   ... 118.65007    126.823395
  137.74821   ]
 [ 90.88955     90.7332      90.76602    ... 106.29597    108.50493
  115.7798    ]
 ...
 [  0.35872427   0.3458393    0.36662653 ...  23.380354    31.088797
   38.938744  ]
 [  0.24737059   0.27054384   0.27587673 ...  19.26295     22.034353
   30.822155  ]
 [  0.28510052   0.30762413   0.27539393 ...  26.240295    31.464735
   36.443123  ]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': '20170421LDL_L02', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': 4365.750022103389, 'y_init': -89.62498249113624, 'z_init': 4000.0, 'n_x': 179, 'n_y': 142, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Fri Apr 21 17:24:12 2017', 'image_name': '', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170421LDL_L02.fsm'} 
, Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[102.06887  102.015755 101.84997  ... 101.92222  118.8075   135.4712  ]
 [102.23148  101.99641  101.785484 ... 118.73554  108.62514   92.96961 ]
 [102.371765 102.30904  102.116    ... 125.76474  135.4335   150.37312 ]
 ...
 [ 51.392715  51.459137  51.29502  ...  63.80772   68.654015  72.258995]
 [ 53.856514  53.817226  53.74031  ...  69.5356    63.362663  59.94399 ]
 [ 50.831997  50.748993  50.635654 ...  68.99248   70.20383   73.61084 ]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': '20170422LDL_L03', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': 7282.250035514433, 'y_init': 327.87501303851553, 'z_init': 4000.0, 'n_x': 287, 'n_y': 106, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Sat Apr 22 16:57:27 2017', 'image_name': 'Default', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170422LDL_L03.fsm'} 
], 'STD': [Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[100.56563  100.43746  100.655556 ... 113.87019  130.67267  130.12047 ]
 [100.524345 100.473595 100.56737  ... 106.732666 122.84013  131.91829 ]
 [100.919174 100.96554  101.06892  ... 117.11189  118.59732  112.632675]
 ...
 [ 87.24211   86.936226  86.670456 ... 107.68202  119.20983  132.96117 ]
 [ 87.515755  87.62498   87.95406  ...  74.24934   82.53994  103.41158 ]
 [ 87.020775  87.24106   87.68801  ...  78.700356  94.35523  113.71412 ]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': 'Image', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': -2266.249990810951, 'y_init': -1309.6249844779575, 'z_init': 4000.0, 'n_x': 75, 'n_y': 126, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Tue Apr 18 19:15:03 2017', 'image_name': 'Default', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170418STD_L01.fsm'} 
, Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[101.894844 102.122215 102.37582  ... 108.37459  114.91064  124.42043 ]
 [101.98463  101.9201   102.075134 ... 120.49213  121.04182  108.73399 ]
 [101.70301  101.74385  101.854485 ... 110.620674 124.91049  144.94037 ]
 ...
 [ 84.49788   84.428     84.40194  ...  80.52712   92.84899   99.83099 ]
 [ 82.908844  82.956955  82.99871  ...  93.9605    84.04683   68.19029 ]
 [ 81.299225  81.46436   81.68425  ...  80.492065  87.23297  106.00833 ]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': '20170418STD_L03', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': 525.8750160187476, 'y_init': 3944.6250122934575, 'z_init': 4000.0, 'n_x': 130, 'n_y': 100, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Tue Apr 18 21:34:15 2017', 'image_name': 'Default', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170418STD_L03.fsm'} 
, Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[104.16222  104.399055 104.81766  ... 129.78352  125.29896  127.66168 ]
 [104.1845   104.35017  104.59667  ... 100.570946 112.29748  104.5784  ]
 [104.23213  104.278755 104.478195 ... 101.34886  120.87445  120.509926]
 ...
 [106.33076  106.470024 106.73167  ... 109.113014 120.45274  127.805046]
 [106.257256 106.27672  106.47709  ... 104.11241  110.68616  145.02014 ]
 [106.32239  106.3202   106.47685  ... 103.12103  103.059875 116.71471 ]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': '20170419STD_L02', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': 5415.500024338563, 'y_init': 2364.500015397866, 'z_init': 4000.0, 'n_x': 197, 'n_y': 125, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Wed Apr 19 15:29:38 2017', 'image_name': 'Default', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170419STD_L02.fsm'} 
], 'EPA': [Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[102.162506 102.20768  102.33103  ... 103.3998   133.57564  132.78831 ]
 [101.934006 101.820946 101.81872  ...  89.18643   82.32647   98.505684]
 [102.250984 101.996895 101.75608  ...  95.217316  99.56977   98.689926]
 ...
 [ 59.419476  59.40246   59.45253  ...  67.783035  73.247665  72.340904]
 [ 59.291172  59.275265  59.470284 ...  59.782402  66.921646  74.762665]
 [ 59.350204  59.409885  59.61621  ...  67.860054  71.12531   94.13546 ]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': '20170419EPA_L03', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': 2986.250023593504, 'y_init': 96.00001490116061, 'z_init': 4000.0, 'n_x': 191, 'n_y': 121, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Wed Apr 19 11:01:00 2017', 'image_name': 'Default', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170419EPA_L03.fsm'} 
, Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[105.09475  105.04887  105.366974 ... 107.11649  102.99424   91.08595 ]
 [104.76829  104.94343  105.12854  ... 114.303444 117.39385  105.55191 ]
 [103.317024 103.37587  103.6764   ... 103.2221   104.533195 105.97731 ]
 ...
 [103.61411  103.58757  103.67997  ...  78.18494   86.83749   94.49371 ]
 [102.8843   102.737976 102.95702  ... 123.78226  127.77606  128.58452 ]
 [102.098305 102.12563  102.436935 ...  90.693794  89.5122    83.86093 ]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': '20170420EPA_L01', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': 3209.000012914339, 'y_init': 1993.875025952855, 'z_init': 4000.0, 'n_x': 105, 'n_y': 210, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Thu Apr 20 17:33:35 2017', 'image_name': 'Default', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170420EPA_L01.fsm'} 
, Spectrum: 
wavelength:
 [4000. 3998. 3996. ...  754.  752.  750.] 
amplitudes: 
 [[ 95.9081    95.937256  95.97436  ... 103.952126 123.30863  149.24287 ]
 [ 94.429726  94.5522    94.72692  ...  78.5463    83.35162  103.35833 ]
 [ 95.19039   94.98259   95.091064 ... 112.38364   96.644714  93.23278 ]
 ...
 [105.68242  105.791405 106.04749  ...  91.40277  110.49822  134.17737 ]
 [105.587585 105.592026 105.73054  ...  98.074196  93.36444  100.17381 ]
 [105.33544  105.62715  105.80969  ...  96.32086  100.53766  112.24974 ]] 
metadata: 
 {'signature': b'PEPE', 'description': 'DataSet - 4DConst3DInterval\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'name': '20170421EPA_L02', 'x_delta': 6.249999751647323, 'y_delta': 6.249999751647323, 'z_delta': -2.0, 'z_start': 4000.0, 'z_end': 750.0, 'z_4d_start': 0.0, 'z_4d_end': 0.0, 'x_init': 3381.1250167638054, 'y_init': -1059.9999880790715, 'z_init': 4000.0, 'n_x': 136, 'n_y': 97, 'n_z': 1626, 'text1': 89, 'text2': 32, 'resolution': 26989, 'text3': 99, 'transmission': 28530, 'text4': 109, 'analyst': '', 'date': 'Fri Apr 21 09:55:23 2017', 'image_name': 'Default', 'instrument_model': 'Spotlight/Frontier FT-NIR', 'instrument_serial_number': '/94002', 'instrument_software_version': '00.01.0060/00.09.1061', 'accumulations': 16, 'detector': 'MCT Array ', 'source': 'MIR', 'beam_splitter': 'OptKBr', 'apodization': 'Filler', 'spectrum_type': 'Ratio (%T)', 'beam_type': 'Sample', 'phase_correction': 'Magnitude', 'ir_accessory': 'Spotlight 400-Enh', 'igram_type': 'Double', 'scan_direction': 'Unidirection', 'background_scans': 750, 'ir_laser_wave_number_unit': '%T', 'filename': '20170421EPA_L02.fsm'} 
]}
In [13]:
metadata = []

for f in filenames:
    name = path.split(f)[1]
    date, slide = name.split('_')
    date, medium = date[:8], date[8:]
    slide = slide[:3]
    metadata.append({'filename': name,
                     'date': date,
                     'medium': medium,
                     'slide': slide})
In [14]:
metadata = pd.DataFrame(metadata)
print(metadata)
print(metadata['medium'].value_counts())
        date             filename medium slide
0   20170418  20170418STD_L01.fsm    STD   L01
1   20170418  20170418STD_L03.fsm    STD   L03
2   20170419  20170419EPA_L03.fsm    EPA   L03
3   20170419  20170419STD_L02.fsm    STD   L02
4   20170420  20170420EDL_L01.fsm    EDL   L01
5   20170420  20170420EPA_L01.fsm    EPA   L01
6   20170420  20170420LDL_L01.fsm    LDL   L01
7   20170421  20170421EDL_L02.fsm    EDL   L02
8   20170421  20170421EPA_L02.fsm    EPA   L02
9   20170421  20170421LDL_L02.fsm    LDL   L02
10  20170422  20170422LDL_L03.fsm    LDL   L03
11  20170424  20170424EDL_L03.fsm    EDL   L03
EPA    3
STD    3
EDL    3
LDL    3
Name: medium, dtype: int64
In [15]:
spectrum = np.array([sp for key, sp in list_spec.items()])
print(type(spectrum[0]))
spectrumAmp=np.array([spAm.amplitudes for spAm in spectrum])
df_spectrum = pd.DataFrame(spectrum)
for s in spectrum:
    print(len(s.amplitudes))
print(spectrumAmp.shape)
<class 'specio.core.util.Spectrum'>
9450
13000
23111
24625
19089
22050
22960
14345
13192
25418
30422
23188
(12,)
In [16]:
# OLD VERSION

# spectrum = np.array([sp for key, sp in list_spec.items()])
# print(type(spectrum[0]))
# spectrumAmp=np.array([spAm.amplitudes for spAm in spectrum])
# df_spectrum = pd.DataFrame(spectrum)
# for s in spectrum:
#     print(len(s.amplitudes))
# print(spectrumAmp.shape)
In [17]:
spectrum = np.zeros((0,1626))
for i in range(4):
     spectrum=np.concatenate( ( spectrum, spectrumAmp[i]) ) 
In [18]:
#type(list_spec)
#list_spec[next(iter(list_spec))]
In [19]:
df_spectrum = pd.DataFrame(spectrum, columns=list_spec[next(iter(list_spec))].wavelength)
# df_spectrum = pd.DataFrame(spectrum, columns=list_spec[0].wavelength)
In [20]:
df_spectrum = pd.concat([metadata[['date','filename', 'medium','slide']],
                         df_spectrum], axis='columns')
In [21]:
df = df_spectrum.set_index(['date','filename', 'medium','slide'])
print(df.head)
<bound method NDFrame.head of                                                4000.0      3998.0      3996.0  \
date     filename            medium slide                                       
20170418 20170418STD_L01.fsm STD    L01    100.565628  100.437462  100.655556   
         20170418STD_L03.fsm STD    L03    100.524345  100.473595  100.567368   
20170419 20170419EPA_L03.fsm EPA    L03    100.919174  100.965538  101.068916   
         20170419STD_L02.fsm STD    L02    100.857765  100.687714  100.738014   
20170420 20170420EDL_L01.fsm EDL    L01    100.038307  100.001404  100.043625   
         20170420EPA_L01.fsm EPA    L01    100.443130  100.286613  100.237091   
         20170420LDL_L01.fsm LDL    L01     99.781319   99.782127   99.919327   
20170421 20170421EDL_L02.fsm EDL    L02     99.920059   99.922890  100.015640   
         20170421EPA_L02.fsm EPA    L02     98.867813   98.909019   98.822746   
         20170421LDL_L02.fsm LDL    L02     99.324081   99.301086   99.487846   
20170422 20170422LDL_L03.fsm LDL    L03     97.648727   97.473358   97.625854   
20170424 20170424EDL_L03.fsm EDL    L03     97.929924   98.087311   98.401871   
NaN      NaN                 NaN    NaN    100.707558  100.897781  101.146111   
                                    NaN    100.523865  100.686882  101.014961   
                                    NaN    101.409744  101.463905  101.606628   
                                    NaN    101.298958  101.167229  101.249306   
                                    NaN    100.134201  100.167328  100.477051   
                                    NaN    101.181587  101.150063  101.210358   
                                    NaN    101.320335  101.342384  101.352371   
                                    NaN    101.108292  101.222267  101.403160   
                                    NaN     98.958916   98.936974   99.172264   
                                    NaN     96.649536   96.657967   96.780930   
                                    NaN     98.037086   97.986336   98.078285   
                                    NaN     98.633995   98.708939   98.927238   
                                    NaN    101.184662  101.305420  101.402985   
                                    NaN    102.573097  102.672607  102.744850   
                                    NaN    103.063278  103.227814  103.421783   
                                    NaN    103.116150  103.084473  103.313644   
                                    NaN    102.316795  102.298477  102.427322   
                                    NaN    100.783340  100.682831  100.949509   
...                                               ...         ...         ...   
                                    NaN    100.777260  100.811867  100.914032   
                                    NaN    102.247948  102.313698  102.731323   
                                    NaN    102.806313  102.853012  103.050240   
                                    NaN    102.898071  102.975212  103.350914   
                                    NaN    102.622520  102.641243  102.992310   
                                    NaN    102.800278  102.625954  102.720184   
                                    NaN    102.998772  103.173286  103.295296   
                                    NaN    102.566078  102.453514  102.457191   
                                    NaN    101.370506  101.531540  101.880219   
                                    NaN    100.370956  100.366791  100.437279   
                                    NaN    102.368912  102.361794  102.657280   
                                    NaN    103.767342  103.766953  104.029892   
                                    NaN    102.990990  102.826973  102.979378   
                                    NaN    102.646469  102.669449  102.939774   
                                    NaN    102.359398  102.274765  102.665016   
                                    NaN    103.968262  104.009453  104.161545   
                                    NaN    104.205429  104.196121  104.477318   
                                    NaN    103.134033  103.315300  103.592659   
                                    NaN    101.543167  101.688293  101.917000   
                                    NaN    102.990639  102.970367  102.994751   
                                    NaN    104.538132  104.676315  105.022194   
                                    NaN    104.057968  104.072235  104.313950   
                                    NaN    102.132980  102.182739  102.388458   
                                    NaN    103.876236  103.878395  104.080772   
                                    NaN    105.723640  105.937202  106.119095   
                                    NaN    105.846237  105.953285  106.168076   
                                    NaN    106.057655  106.088699  106.293068   
                                    NaN    106.330757  106.470024  106.731667   
                                    NaN    106.257256  106.276718  106.477089   
                                    NaN    106.322388  106.320198  106.476852   

                                               3994.0      3992.0      3990.0  \
date     filename            medium slide                                       
20170418 20170418STD_L01.fsm STD    L01    101.042213  101.127029  101.041862   
         20170418STD_L03.fsm STD    L03    100.859825  101.011810  100.881363   
20170419 20170419EPA_L03.fsm EPA    L03    101.187706  101.240456  101.315536   
         20170419STD_L02.fsm STD    L02    100.876060  101.003120  101.168861   
20170420 20170420EDL_L01.fsm EDL    L01    100.264618  100.468994  100.374901   
         20170420EPA_L01.fsm EPA    L01    100.350197  100.373093  100.387566   
         20170420LDL_L01.fsm LDL    L01    100.200516  100.322189  100.209435   
20170421 20170421EDL_L02.fsm EDL    L02     99.979538   99.937737  100.053764   
         20170421EPA_L02.fsm EPA    L02     98.903679   99.061241   99.115631   
         20170421LDL_L02.fsm LDL    L02     99.790405  100.009796  100.049370   
20170422 20170422LDL_L03.fsm LDL    L03     98.008072   98.248367   98.330795   
20170424 20170424EDL_L03.fsm EDL    L03     98.482269   98.331657   98.288475   
NaN      NaN                 NaN    NaN    101.172783  101.122574  101.068527   
                                    NaN    101.265068  101.418175  101.441475   
                                    NaN    101.688332  101.782608  101.840454   
                                    NaN    101.422386  101.529556  101.549568   
                                    NaN    100.653099  100.691956  100.718109   
                                    NaN    101.335983  101.351372  101.378624   
                                    NaN    101.296646  101.226707  101.358971   
                                    NaN    101.438881  101.384811  101.409660   
                                    NaN     99.350510   99.318413   99.282066   
                                    NaN     96.880531   96.841644   96.969887   
                                    NaN     98.170647   98.209190   98.266426   
                                    NaN     98.998413   98.803551   98.630127   
                                    NaN    101.494896  101.456848  101.366470   
                                    NaN    102.736801  102.739510  102.837883   
                                    NaN    103.618141  103.686699  103.570320   
                                    NaN    103.545265  103.581879  103.541756   
                                    NaN    102.606453  102.640564  102.613075   
                                    NaN    101.157631  101.114922  101.183067   
...                                               ...         ...         ...   
                                    NaN    101.101219  101.146675  101.119804   
                                    NaN    103.064972  103.087975  103.013954   
                                    NaN    103.156837  103.198341  103.283920   
                                    NaN    103.603951  103.672760  103.898476   
                                    NaN    103.319633  103.326538  103.366898   
                                    NaN    102.986526  103.020821  102.987846   
                                    NaN    103.319916  103.391968  103.650070   
                                    NaN    102.541862  102.734886  102.966606   
                                    NaN    102.053375  102.041237  102.092140   
                                    NaN    100.614357  100.803871  100.943771   
                                    NaN    103.046471  103.248253  103.228333   
                                    NaN    104.169853  104.152046  104.284393   
                                    NaN    103.177605  103.156570  103.172562   
                                    NaN    103.141785  103.073181  103.019325   
                                    NaN    103.146568  103.430466  103.598946   
                                    NaN    104.353561  104.509460  104.497269   
                                    NaN    104.772896  104.916801  104.975418   
                                    NaN    103.792297  103.807617  103.782814   
                                    NaN    102.016884  102.050018  102.049782   
                                    NaN    103.211342  103.394142  103.503998   
                                    NaN    105.421814  105.566093  105.452629   
                                    NaN    104.513428  104.505569  104.538742   
                                    NaN    102.558014  102.557755  102.636276   
                                    NaN    104.273811  104.315712  104.451172   
                                    NaN    106.187050  106.149406  106.050110   
                                    NaN    106.332436  106.404892  106.552223   
                                    NaN    106.691406  107.103096  107.253860   
                                    NaN    106.943954  107.062592  107.098076   
                                    NaN    106.683800  106.905350  107.017441   
                                    NaN    106.709534  106.852783  106.942635   

                                               3988.0      3986.0      3984.0  \
date     filename            medium slide                                       
20170418 20170418STD_L01.fsm STD    L01    100.910751  100.865280  100.962021   
         20170418STD_L03.fsm STD    L03    100.574974  100.511772  100.669250   
20170419 20170419EPA_L03.fsm EPA    L03    101.130440  100.767326  100.745483   
         20170419STD_L02.fsm STD    L02    101.041557  100.821655  100.896675   
20170420 20170420EDL_L01.fsm EDL    L01    100.055763   99.954872  100.125397   
         20170420EPA_L01.fsm EPA    L01    100.381561  100.376938  100.457397   
         20170420LDL_L01.fsm LDL    L01    100.021652   99.910301   99.854286   
20170421 20170421EDL_L02.fsm EDL    L02    100.077660   99.982674   99.839264   
         20170421EPA_L02.fsm EPA    L02     98.796257   98.382439   98.376617   
         20170421LDL_L02.fsm LDL    L02     99.777679   99.586403   99.624626   
20170422 20170422LDL_L03.fsm LDL    L03     98.171387   97.917404   97.820259   
20170424 20170424EDL_L03.fsm EDL    L03     98.131828   97.962547   98.051163   
NaN      NaN                 NaN    NaN    100.795059  100.564850  100.699104   
                                    NaN    101.115059  100.809456  100.893379   
                                    NaN    101.699097  101.661537  101.830284   
                                    NaN    101.390434  101.261772  101.378426   
                                    NaN    100.511581  100.294182  100.397125   
                                    NaN    101.270973  101.156845  101.256187   
                                    NaN    101.380615  101.311760  101.436279   
                                    NaN    101.434792  101.451614  101.458961   
                                    NaN     99.134193   99.009178   99.103783   
                                    NaN     97.063652   96.950272   96.899689   
                                    NaN     98.125877   98.046371   98.251450   
                                    NaN     98.455757   98.321823   98.540588   
                                    NaN    101.267822  101.237198  101.302116   
                                    NaN    102.739700  102.484962  102.409019   
                                    NaN    103.238609  103.078979  103.230988   
                                    NaN    103.394341  103.272560  103.316582   
                                    NaN    102.413216  102.069626  102.015404   
                                    NaN    101.117531  100.980415  101.089264   
...                                               ...         ...         ...   
                                    NaN    100.978111  100.748283  100.718498   
                                    NaN    102.831612  102.727715  102.932297   
                                    NaN    103.087715  102.788193  102.853317   
                                    NaN    103.860626  103.579712  103.549484   
                                    NaN    103.366478  103.199524  103.117081   
                                    NaN    102.901306  102.910378  103.046501   
                                    NaN    103.538994  103.199615  103.242645   
                                    NaN    102.842705  102.528267  102.578934   
                                    NaN    101.882034  101.550720  101.521309   
                                    NaN    100.809067  100.624687  100.647217   
                                    NaN    102.857292  102.508919  102.616096   
                                    NaN    104.121857  103.679535  103.665115   
                                    NaN    103.113266  103.075020  103.266174   
                                    NaN    102.884338  102.757912  102.936836   
                                    NaN    103.413574  103.101234  103.070282   
                                    NaN    104.206413  104.012299  104.179771   
                                    NaN    104.719467  104.321556  104.263184   
                                    NaN    103.485748  103.117172  103.178848   
                                    NaN    101.703392  101.331001  101.460556   
                                    NaN    103.404640  103.125984  103.024109   
                                    NaN    105.042053  104.697525  104.769348   
                                    NaN    104.496307  104.325920  104.499146   
                                    NaN    102.570503  102.329849  102.377327   
                                    NaN    104.364967  104.014046  104.025848   
                                    NaN    105.791168  105.635605  105.882988   
                                    NaN    106.526405  106.292870  106.261147   
                                    NaN    106.931076  106.535164  106.412582   
                                    NaN    106.924377  106.762016  106.858765   
                                    NaN    106.706062  106.422119  106.552071   
                                    NaN    106.916428  106.779396  106.810669   

                                               3982.0     ...          768.0   \
date     filename            medium slide                 ...                   
20170418 20170418STD_L01.fsm STD    L01    101.082581     ...       97.847168   
         20170418STD_L03.fsm STD    L03    100.680412     ...       88.620834   
20170419 20170419EPA_L03.fsm EPA    L03    100.949631     ...       97.180229   
         20170419STD_L02.fsm STD    L02    100.990318     ...       97.403694   
20170420 20170420EDL_L01.fsm EDL    L01    100.189461     ...       91.867386   
         20170420EPA_L01.fsm EPA    L01    100.403809     ...       91.011871   
         20170420LDL_L01.fsm LDL    L01     99.849792     ...       92.671143   
20170421 20170421EDL_L02.fsm EDL    L02     99.756645     ...       89.964493   
         20170421EPA_L02.fsm EPA    L02     98.610786     ...      104.043015   
         20170421LDL_L02.fsm LDL    L02     99.642006     ...       95.606041   
20170422 20170422LDL_L03.fsm LDL    L03     97.902710     ...       92.277618   
20170424 20170424EDL_L03.fsm EDL    L03     98.105408     ...       96.227180   
NaN      NaN                 NaN    NaN    100.895401     ...      108.243217   
                                    NaN    101.052010     ...       93.163918   
                                    NaN    101.912880     ...      104.118797   
                                    NaN    101.407669     ...       87.206390   
                                    NaN    100.556946     ...       94.742813   
                                    NaN    101.448196     ...       90.703300   
                                    NaN    101.612427     ...       96.453201   
                                    NaN    101.401611     ...       91.815041   
                                    NaN     99.291458     ...       92.194069   
                                    NaN     96.914688     ...       82.858711   
                                    NaN     98.296463     ...       94.943924   
                                    NaN     98.793625     ...       87.688866   
                                    NaN    101.376289     ...      101.239548   
                                    NaN    102.508377     ...       86.944206   
                                    NaN    103.374344     ...       99.760765   
                                    NaN    103.286270     ...       89.517639   
                                    NaN    102.244118     ...       96.053947   
                                    NaN    101.146576     ...       88.004372   
...                                               ...     ...             ...   
                                    NaN    100.902367     ...       87.174370   
                                    NaN    103.077293     ...      100.824036   
                                    NaN    103.035210     ...       90.893433   
                                    NaN    103.518913     ...       92.329071   
                                    NaN    103.154640     ...       89.028191   
                                    NaN    103.086075     ...       98.479439   
                                    NaN    103.512543     ...       82.885292   
                                    NaN    102.883621     ...       92.459297   
                                    NaN    101.629356     ...       81.153015   
                                    NaN    100.730591     ...       92.652802   
                                    NaN    102.881889     ...       87.379326   
                                    NaN    103.981987     ...       95.537773   
                                    NaN    103.372910     ...       87.237984   
                                    NaN    103.144638     ...       83.719475   
                                    NaN    103.168159     ...       93.469635   
                                    NaN    104.433418     ...       93.624466   
                                    NaN    104.366058     ...       87.932289   
                                    NaN    103.468414     ...       94.257248   
                                    NaN    101.780052     ...       85.009865   
                                    NaN    103.172470     ...       77.939697   
                                    NaN    104.979858     ...       81.837387   
                                    NaN    104.841652     ...       94.542046   
                                    NaN    102.612976     ...       99.870026   
                                    NaN    104.471138     ...       93.032730   
                                    NaN    106.185516     ...       84.111847   
                                    NaN    106.413635     ...       94.299927   
                                    NaN    106.549843     ...       79.366203   
                                    NaN    106.876579     ...       94.780693   
                                    NaN    106.708603     ...       86.119209   
                                    NaN    106.930885     ...       89.656952   

                                               766.0       764.0       762.0   \
date     filename            medium slide                                       
20170418 20170418STD_L01.fsm STD    L01     99.934502   98.916954   97.683571   
         20170418STD_L03.fsm STD    L03     88.410263   88.908142   88.539413   
20170419 20170419EPA_L03.fsm EPA    L03     97.375801  102.663254  114.905739   
         20170419STD_L02.fsm STD    L02     91.576401   89.823914   91.289764   
20170420 20170420EDL_L01.fsm EDL    L01     87.589760   87.946770   98.988518   
         20170420EPA_L01.fsm EPA    L01     92.567955   86.593269   83.686249   
         20170420LDL_L01.fsm LDL    L01     97.254547  100.758766  100.427223   
20170421 20170421EDL_L02.fsm EDL    L02     90.353584   97.386131  101.835999   
         20170421EPA_L02.fsm EPA    L02    105.049744  101.846985   96.491173   
         20170421LDL_L02.fsm LDL    L02     87.655197   84.711105   86.612267   
20170422 20170422LDL_L03.fsm LDL    L03     89.417854   90.469185   94.064812   
20170424 20170424EDL_L03.fsm EDL    L03     94.055908   97.769157  100.939896   
NaN      NaN                 NaN    NaN    103.757820   98.143608  102.016747   
                                    NaN     93.197144   89.236572   85.776962   
                                    NaN    104.342163  112.134460  119.816986   
                                    NaN     87.721451   88.294357   91.709450   
                                    NaN    100.480103  101.994179  107.915695   
                                    NaN     92.288826   91.804420   84.179291   
                                    NaN     92.578758   91.326553   91.119362   
                                    NaN     95.910400   99.541359   94.143921   
                                    NaN     96.189926  101.902031  104.882935   
                                    NaN     86.562057   89.648300   91.527390   
                                    NaN     94.446526   96.256729   95.144218   
                                    NaN     83.453026   83.650505   87.118317   
                                    NaN    102.612473  100.282814   96.952637   
                                    NaN     89.071732   93.803757   99.715996   
                                    NaN     99.210052  104.921722  105.551941   
                                    NaN     89.289810   88.998016   91.661736   
                                    NaN    103.091301  117.383934  121.030472   
                                    NaN     86.677589   90.121895   98.115494   
...                                               ...         ...         ...   
                                    NaN     91.290176   95.661148   94.256355   
                                    NaN     96.268364   92.197662   97.155876   
                                    NaN     92.403267   94.108315   88.985130   
                                    NaN     92.321854   95.362221   96.325684   
                                    NaN     95.365204   95.285606   95.824982   
                                    NaN     96.746750  100.504051   98.223961   
                                    NaN     79.937317   80.184105   82.810097   
                                    NaN     84.320915   83.680252   92.836510   
                                    NaN     81.095695   82.739487   85.033768   
                                    NaN     97.657761   96.038300   91.474518   
                                    NaN     85.903908   79.372910   78.047264   
                                    NaN     98.187317   92.996117   85.032242   
                                    NaN     97.028976  102.408615   98.007240   
                                    NaN     89.724762   95.743675   97.689003   
                                    NaN     92.720810   87.362144   88.195557   
                                    NaN     96.190445  100.837036   98.078812   
                                    NaN     86.791588   90.860252   87.718681   
                                    NaN     87.585037   79.162239   79.784500   
                                    NaN     84.847832   86.460388   91.257790   
                                    NaN     79.889091   84.901070   95.860443   
                                    NaN     81.122726   83.827202   84.746201   
                                    NaN     88.900993   91.196976   96.487717   
                                    NaN    100.818489   94.008492   92.473572   
                                    NaN     93.969879   86.749847   71.764000   
                                    NaN     82.870277   83.061073   85.660034   
                                    NaN     84.158897   82.856895   87.493027   
                                    NaN     84.372948   89.287621   91.708710   
                                    NaN    106.678513  111.632858  100.797249   
                                    NaN     78.669502   78.765411   83.522842   
                                    NaN     96.044250  103.532494  101.821732   

                                               760.0       758.0       756.0   \
date     filename            medium slide                                       
20170418 20170418STD_L01.fsm STD    L01     92.842155   96.027298   96.088387   
         20170418STD_L03.fsm STD    L03     86.191490   84.405281   87.334167   
20170419 20170419EPA_L03.fsm EPA    L03    118.503342  116.997360  119.955429   
         20170419STD_L02.fsm STD    L02     88.808784   95.976746  111.838791   
20170420 20170420EDL_L01.fsm EDL    L01    107.015526  107.126251  110.778641   
         20170420EPA_L01.fsm EPA    L01     85.868645   97.246002  110.951599   
         20170420LDL_L01.fsm LDL    L01     96.501488   95.518997   95.328941   
20170421 20170421EDL_L02.fsm EDL    L02     95.810768   91.124649   94.859505   
         20170421EPA_L02.fsm EPA    L02     94.230507  103.275208  109.447952   
         20170421LDL_L02.fsm LDL    L02     80.581963   82.191971   89.463554   
20170422 20170422LDL_L03.fsm LDL    L03     95.721458   89.059097   89.150780   
20170424 20170424EDL_L03.fsm EDL    L03     94.171326   84.503754   87.896156   
NaN      NaN                 NaN    NaN    110.541435  112.555496  112.099350   
                                    NaN     86.177628   86.120354   80.234840   
                                    NaN    122.106201  128.222351  130.554031   
                                    NaN     96.725685   95.878967  106.070190   
                                    NaN    109.565353  100.807991   98.328598   
                                    NaN     78.364723   81.415543   83.699532   
                                    NaN     85.425476   82.000771   88.244247   
                                    NaN     88.442635   89.258858   89.521736   
                                    NaN    102.331520  104.790810  105.319313   
                                    NaN     95.512703   96.776672   92.712936   
                                    NaN     96.771492  102.043358  105.163612   
                                    NaN     92.217873   93.079643   94.566597   
                                    NaN    101.188896  102.033966   94.247391   
                                    NaN    102.025940  100.503807  100.743172   
                                    NaN     96.492523   96.473763  110.565521   
                                    NaN     97.152466  101.936752  103.548485   
                                    NaN    110.498100  100.448959   96.733490   
                                    NaN    106.968147   99.374992   85.719589   
...                                               ...         ...         ...   
                                    NaN     88.306602   86.367424   84.838478   
                                    NaN    105.982300  109.171600  101.042694   
                                    NaN     83.297440   84.626945   87.911278   
                                    NaN     97.015106   94.567345   86.227242   
                                    NaN    103.954407  103.351685   96.190948   
                                    NaN     91.974136  100.171455  111.190247   
                                    NaN     82.426643   85.251747   84.637634   
                                    NaN    102.717896  107.364151  111.667023   
                                    NaN     83.235626   86.032768   97.710838   
                                    NaN    100.250175  114.536087  114.504799   
                                    NaN     87.425621   95.255112   93.280022   
                                    NaN     84.818810   91.358185  104.639030   
                                    NaN    103.403496  110.116470   97.084328   
                                    NaN     93.761063   95.600655  101.437691   
                                    NaN     94.011917   94.566177   87.154015   
                                    NaN     96.323486  102.080765  103.145172   
                                    NaN     86.979218   92.121437   93.472984   
                                    NaN     95.074821  106.094154  108.195168   
                                    NaN     88.581253   80.782463   91.900406   
                                    NaN    101.654381  102.141472  109.681610   
                                    NaN     88.912285   89.839912   99.810211   
                                    NaN     98.597908   97.906975  103.217842   
                                    NaN    103.089981  103.564880   99.391121   
                                    NaN     71.467560   85.254601   97.048798   
                                    NaN     85.424843   77.223602   79.307907   
                                    NaN     91.339355  101.145294  102.862724   
                                    NaN     91.835510   85.769707   83.440140   
                                    NaN     87.645233   91.677406  101.393486   
                                    NaN     94.490059   97.960152   99.420647   
                                    NaN     98.119583  106.574509  114.233269   

                                               754.0       752.0       750.0   
date     filename            medium slide                                      
20170418 20170418STD_L01.fsm STD    L01    113.870193  130.672668  130.120468  
         20170418STD_L03.fsm STD    L03    106.732666  122.840134  131.918289  
20170419 20170419EPA_L03.fsm EPA    L03    117.111893  118.597321  112.632675  
         20170419STD_L02.fsm STD    L02    112.974892   95.770256   90.075630  
20170420 20170420EDL_L01.fsm EDL    L01    111.859688  124.256477  139.328354  
         20170420EPA_L01.fsm EPA    L01    106.214729   95.746674  102.821426  
         20170420LDL_L01.fsm LDL    L01    105.457169  120.860954  134.041229  
20170421 20170421EDL_L02.fsm EDL    L02     91.000809   98.432159  115.355316  
         20170421EPA_L02.fsm EPA    L02    104.784416   98.080559  112.955231  
         20170421LDL_L02.fsm LDL    L02     98.798721  114.556267  129.300308  
20170422 20170422LDL_L03.fsm LDL    L03     96.951324  104.087776  129.143875  
20170424 20170424EDL_L03.fsm EDL    L03     93.599846   90.318748  102.521927  
NaN      NaN                 NaN    NaN    120.949921  134.045258  134.507690  
                                    NaN     77.837006   85.999451   96.042374  
                                    NaN    126.487251  122.507408  125.694550  
                                    NaN    114.774338  110.014786  108.176598  
                                    NaN    116.974342  136.313232  142.106888  
                                    NaN     86.489349   89.476707  110.592361  
                                    NaN     98.255325  115.094772  112.032875  
                                    NaN     89.980728   95.863831  133.690338  
                                    NaN     92.559456   93.372200  111.725449  
                                    NaN     89.670258  104.136192  129.500397  
                                    NaN    102.873634   88.581467   93.870941  
                                    NaN     99.420502   95.210159  109.786858  
                                    NaN     86.744728   92.390434  110.039902  
                                    NaN     96.428345   91.777588  102.133469  
                                    NaN    115.881119  111.549980  124.439857  
                                    NaN    108.461761  114.816826  103.633095  
                                    NaN    107.314552  127.992340  144.923645  
                                    NaN     87.516823   83.144257   79.988937  
...                                               ...         ...         ...  
                                    NaN     80.131302   91.170433  110.405586  
                                    NaN    102.089340  103.251915  100.954506  
                                    NaN    100.101814  106.612007  115.218018  
                                    NaN     91.454987  109.662926  130.148880  
                                    NaN     87.868607   86.699188  100.751404  
                                    NaN    111.961784  118.116463  139.369934  
                                    NaN     88.090385   96.863525  116.877663  
                                    NaN    113.579231  118.856903  126.939423  
                                    NaN    103.149506   97.596558   92.920403  
                                    NaN    110.349564  114.752075  126.208321  
                                    NaN    102.702995  114.197426  110.894417  
                                    NaN    108.639786  108.381905  120.923454  
                                    NaN     91.397369   94.453857   99.098465  
                                    NaN    104.611053  121.836426  144.519989  
                                    NaN     83.709595   84.987785   98.436295  
                                    NaN    109.984978  107.802322  108.511925  
                                    NaN     80.918518   69.891281   87.796272  
                                    NaN    106.367905  109.786850  101.845345  
                                    NaN    101.259224  106.773705  124.803596  
                                    NaN    117.546150  120.042358  127.797256  
                                    NaN    101.262184   97.645584  110.235054  
                                    NaN    116.192940  124.565170  137.355240  
                                    NaN     93.998566   83.536140   85.828781  
                                    NaN     98.797340  100.294891  110.495911  
                                    NaN     97.077164  110.824722  118.402489  
                                    NaN    110.327568  126.226738  148.152344  
                                    NaN     98.902176  105.689407   98.046310  
                                    NaN    109.113014  120.452744  127.805046  
                                    NaN    104.112411  110.686157  145.020142  
                                    NaN    103.121033  103.059875  116.714706  

[70186 rows x 1626 columns]>
In [22]:
import matplotlib.pyplot as plt
%matplotlib inline
plt.plot(df.groupby('medium').mean().T)
plt.show()
In [23]:
pca = PCA(n_components=6)
In [24]:
pca.fit(spectrum)
Out[24]:
PCA(copy=True, iterated_power='auto', n_components=6, random_state=None,
  svd_solver='auto', tol=0.0, whiten=False)
In [25]:
wavelength=list_spec[next(iter(list_spec))].wavelength
#wavelength=list_spec[0].wavelength
plt.plot(wavelength,pca.components_[:6].T)
plt.show()
In [26]:
plt.plot(pca.explained_variance_ratio_[0:5])
plt.show()
In [27]:
print(pca.explained_variance_ratio_[0:5])
[0.71655348 0.2016908  0.02776846 0.02209929 0.01167137]
In [28]:
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline
In [29]:
pca_norm = make_pipeline(StandardScaler(), PCA(n_components=6, whiten=True))
pca = PCA(n_components=6, whiten=True)
X_r = pca.fit_transform(spectrum)
X_r = pca_norm.fit_transform(spectrum)
In [30]:
print(X_r.shape)
(70186, 6)
In [31]:
plt.plot(wavelength,pca.components_[0].T)
plt.axvspan(1000, 1120, alpha=0.2, color='grey')
plt.show()
In [32]:
plt.plot(wavelength,pca.components_[1].T)
plt.axvspan(750, 898, alpha=0.2, color='grey')
plt.show()
In [33]:
plt.plot(wavelength,pca.components_[2].T)
plt.axvspan(750, 898, alpha=0.2, color='grey')
plt.show()
In [34]:
plt.plot(wavelength,pca.components_[3].T)
plt.axvspan(750, 898, alpha=0.2, color='grey')
plt.show()
In [35]:
plt.plot(wavelength,pca.components_[4].T)
plt.axvspan(2400, 2450, alpha=0.2, color='grey')
plt.show()
In [36]:
plt.plot(wavelength,pca.components_[5].T)
plt.axvspan(1000, 1200, alpha=0.2, color='grey')
plt.show()
In [37]:
for medium in metadata['medium'].unique() :
    mask = metadata['medium'] == medium
    plt.scatter(X_r[mask, 0], X_r[mask, 1],
                label=medium, s=5, alpha=0.7)
plt.legend()
plt.show()
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-37-b85e9a59e20e> in <module>()
      1 for medium in metadata['medium'].unique() :
      2     mask = metadata['medium'] == medium
----> 3     plt.scatter(X_r[mask, 0], X_r[mask, 1],
      4                 label=medium, s=5, alpha=0.7)
      5 plt.legend()

IndexError: boolean index did not match indexed array along dimension 0; dimension is 70186 but corresponding boolean dimension is 12
In [ ]:
plt.scatter(X_r[:, 0], X_r[:, 1])
plt.show()
In [ ]:
df_spectrum