Merge branch '40-read-nb2' of gitlab.epfl.ch:spc/gdat into 40-read-nb2

forgot to pull first with a rebase from gitlab gui

.gitlab-ci.yml

@@ -5,50 +5,62 @@ include:
 
 variables:
   TBXFOLDER: matlab
-#  MATFIRST: 803
   MATFIRST: 902
   MATVER: 906
 
 # no build for gdat, so don't define jobs for build phase
 
 # test jobs
-#test-matlab-9.8.0:
-#  extends: .test-template
-#  tags:
-#  - lac8-gituser
-#  variables:
-#    MATLABCMD: matlab980
-#    TESTCASE: basic-tcv
-
-test-matlab-9.6.0:
+TCV-960:
   extends: .test-template
   tags:
   - lac8-gituser
   variables:
     MATLABCMD: matlab960
-    TESTCASE: basic-tcv
-
-test-matlab-9.2.0:
+    TESTCASE: tcv
+  artifacts:
+   paths:
+    - test_${TBXTARGET}_${TESTCASE}_2019a_cov.xml
+   reports:
+    junit: test_${TBXTARGET}_${TESTCASE}_2019a.xml
+    cobertura: test_${TBXTARGET}_${TESTCASE}_2019a_cov.xml
+
+TCV_IMAS-960:
+  extends: .test-template
+  tags:
+  - lac8-gituser
+  variables:
+    MATLABCMD: matlab960
+    TESTCASE: tcv_imas
+  artifacts:
+   paths:
+    - test_${TBXTARGET}_${TESTCASE}_2019a_cov.xml
+   reports:
+    junit: test_${TBXTARGET}_${TESTCASE}_2019a.xml
+    cobertura: test_${TBXTARGET}_${TESTCASE}_2019a_cov.xml
+
+TCV-920:
   extends: .test-template
   variables:
     MATLABCMD: matlab920
-    TESTCASE: basic-tcv
+    TESTCASE: tcv
+  artifacts:
+   reports:
+    junit: test_${TBXTARGET}_${TESTCASE}_2017a.xml
 
-#test-matlab-9.8.0-all:
-#  extends: test-matlab-9.8.0
-#  variables:
-#     TESTCASE: all
-#  only:
-#  - schedules
-#  - web
 
-test-matlab-9.6.0-all:
-  extends: test-matlab-9.6.0
+TCV_IMAS-920:
+  extends: .test-template
   variables:
-     TESTCASE: all
-  only:
-  - schedules
-  - web
+    MATLABCMD: matlab920
+    TESTCASE: tcv_imas
+  artifacts:
+   reports:
+    junit: test_${TBXTARGET}_${TESTCASE}_2017a.xml
+
+# Post test report collection
+code-coverage-960:
+ extends: .code-coverage-960-template
 
 # Staging, testing and deployment for releases
 pre-deployment-staging:
@@ -71,21 +83,6 @@ test-staging-920:
  variables:
   MATLABCMD: matlab920
 
-#test-staging-900:
-# extends: .test-staging-template
-# variables:
-#  MATLABCMD: matlab900
-
-#test-staging-850:
-# extends: .test-staging-template
-# variables:
-#  MATLABCMD: matlab850
-
-#test-staging-830:
-# extends: .test-staging-template
-# variables:
-#  MATLABCMD: matlab830
-
 # pre-deployment
 pre-deployment-checks:
  extends: .pre-deployment-checks-template
@@ -100,9 +97,3 @@ test-post-deploy-matlab-9.2.0:
   variables:
     MATLABCMD: matlab920
     TESTCASE: basic-tcv
-
-#test-post-deploy-matlab-8.5.0:
-#  extends: .test-post-deploy-template
-#  variables:
-#    MATLABCMD: matlab850
-#    TESTCASE: basic-tcv

matlab/TCV/gdat_tcv.m

@@ -3119,12 +3119,7 @@ elseif strcmp(mapping_for_tcv.method,'switchcase')
     %
     aaa=which('get_scd_mems');
     if isempty(aaa)
-      if exist('/home/sauter/NoTivoli/rtccode_develop/tools')
-        addpath('/home/sauter/NoTivoli/rtccode_develop/tools','-end');
-      else
-        warning('Cannot add path for get_scd_mems')
-        return
-      end
+      error('no path for get_scd_mems')
     end
 
     aa = get_scd_mems(shot);

matlab/gdatpaths.m

@@ -11,7 +11,7 @@ end
 a=which('gdat');
 path_gdat = fileparts(a);
 if ~findstr(path,[path_gdat,':'])
-  warning('Folder %s containing gdat.m is not in your path!!',path_gdat);
+  error('Folder %s containing gdat.m is not in your path!!',path_gdat);
 end
 
 if isempty(a), error('gdat is not in the MATLAB path'); end
@@ -22,7 +22,7 @@ a = fileparts(a);
 machines=[{'JET'} {'TCV'} {'AUG'} {'D3D'} {'KSTAR'} {'TCV_IMAS'} {'AUG_IMAS'} {'CHDF'} {'IMAS'}];
 
 add_paths = cell(1,length(machines));
-for i=1:length(machines)
+for i=1:numel(machines)
   add_paths{i}=fullfile(a,machines{i});
 end
 if ~remove

matlab/run_gdat_tests.m

-function [passed,results] = run_gdat_tests(test_case)
-% test runner for gdat
+function [passed,results] = run_gdat_tests(test_case,coverage_report)
+% Test runner for generic toolbox tests
 
-% F. Felici, EPFL federico.felici@epfl.ch
-
-if nargin==0 || isempty(test_case)
-  test_case = 'basic-tcv'; % default
+if nargin==0
+  test_case = ''; % default
+end
+if nargin < 2
+  coverage_report = false; % default
 end
-test_case = lower(test_case);
 
+needXML=~isempty(getenv('GITLAB_CI')) && ~verLessThan('matlab','8.6.0');
+needCOV=~isempty(getenv('GITLAB_CI')) && ~verLessThan('matlab','9.6.0') || coverage_report;
+
+%% Default outputs
+passed=false; results=[]; % default outputs
+ 
 %% Import some classes we need
 import matlab.unittest.selectors.HasTag;
 import matlab.unittest.constraints.ContainsSubstring;
 import matlab.unittest.selectors.HasName;
+if needXML
+  import matlab.unittest.plugins.XMLPlugin;
+end
+
+%% Paths
+% add path of toolbox to make sure it is in the path
+tbxpath = fileparts(mfilename('fullpath'));
+addpath(tbxpath); % add default path
+projectpath = getenv('CI_PROJECT_DIR');
+if isempty(projectpath), projectpath = tbxpath; end
+
+% add additional paths present in a local setpaths_* file
+setpathsfile = dir(fullfile(tbxpath,'setpaths_*'));
+if numel(setpathsfile)==1
+  fname = fullfile(tbxpath,setpathsfile.name);
+  fprintf('adding extra paths by calling %s\n',fname);
+  run(fname); % run file to add paths
+elseif numel(setpathsfile)>1
+  error('multiple setpaths_* files found!');
+end
 
-%% populate suite
-% add path with tests
-addpath(genpath(fullfile(fileparts(mfilename('fullpath')),'tests')));
-addpath(genpath(fullfile(fileparts(mfilename('fullpath')),'TCV_IMAS')));
+%% Name
+tbxname = 'gdat';
 
-% aug cannot be tested without a tunnel created
-% $$$ suite_all = [matlab.unittest.TestSuite.fromClass(?test_requestnames_tcv),...
-% $$$   matlab.unittest.TestSuite.fromClass(?test_requestnames_aug),...
-% $$$   matlab.unittest.TestSuite.fromClass(?test_tcv_get_ids)];
+%% Generate test suite
+testspath = fullfile(tbxpath,'tests');
 
-suite_all = [matlab.unittest.TestSuite.fromClass(?test_requestnames_tcv),...
-  matlab.unittest.TestSuite.fromClass(?test_tcv_get_ids)];
+lastwarn('','');
 
-switch test_case
-  case 'all'
+suite_all = matlab.unittest.TestSuite.fromFolder(testspath);
+if isempty(test_case)
+  % list all tags
+  fprintf('Available (possibly overlapping) test_case tags:\n')
+  disp([{'all'};unique([suite_all.Tags])']);
+  return
+end
+if coverage_report
+  assert(~verLessThan('matlab','9.6.0'),'coverage_report=true needs MATLAB 9.6.0 or later');
+  assert( strcmpi(test_case,'coverage'),'coverage_report=true should only be used with test_case=''coverage''');
+end
+
+[~,s] = lastwarn();
+if isequal(s,'MATLAB:unittest:TestSuite:FileExcluded')
+  fprintf('File excluded during test suite creation - possible syntax errors in a test class');
+  return
+end
+ 
+switch lower(test_case)
+  case {'all','coverage'}
     suite = suite_all; % run all
-  case 'basic-tcv'
-    s = ~HasTag('slow') & HasName(ContainsSubstring('tcv'));
-    suite = suite_all.selectIf(s);
-  case 'imas'
-    s = HasTag('imas');
-    suite = suite_all.selectIf(s);
-  case 'tcv'
-    s = HasName(ContainsSubstring('tcv'));
-    suite = suite_all.selectIf(s);
-  case 'aug'
-    s = HasName(ContainsSubstring('aug'));
+  otherwise
+    s = HasTag(test_case);
     suite = suite_all.selectIf(s);
+end
 
-  otherwise
-    error('unknown test_case %s',test_case)
+if isempty(suite)
+  fprintf('\nEmpty test suite returned for TestTag=''%s''\n',test_case); return;
 end
 
 %% run it
 fprintf('Start test case: %s\n%s\n\n',test_case,datestr(now));
-results = run(suite);
+runner = matlab.unittest.TestRunner.withTextOutput;
+
+%% Reports
+if needXML || needCOV
+  prefix = sprintf('test_%s',tbxname);
+  suffix = version('-release');
+end
+if needXML
+  % Add some JUnit XML file with tests results
+  xmlFile = fullfile(projectpath,sprintf('%s_%s_%s.xml',prefix,test_case,suffix));
+  fprintf('\tGenerating JUnit XML report at %s\n',xmlFile);
+  p = XMLPlugin.producingJUnitFormat(xmlFile);
+  runner.addPlugin(p)
+end
+if needCOV
+  % Add some code coverage report
+  switch lower(test_case)
+    case 'coverage'
+      % Produce HTML report
+      reportFolder = fullfile(projectpath,sprintf('%s_%s_cov',prefix,suffix));
+      reportFormat = matlab.unittest.plugins.codecoverage.CoverageReport(reportFolder);
+    otherwise
+      % Produce XML file in Cobertura format (for Gitlab MR viz.)
+      xmlFile = fullfile(projectpath,sprintf('%s_%s_%s_cov.xml',prefix,test_case,suffix));
+      reportFormat = matlab.unittest.plugins.codecoverage.CoberturaFormat(xmlFile);
+  end
+  p = matlab.unittest.plugins.CodeCoveragePlugin.forFolder(fileparts(mfilename('fullpath')),...
+    'IncludingSubfolders',true,'Producing',reportFormat);
+  runner.addPlugin(p)
+end
+results = runner.run(suite);
 disp(table(results));
 fprintf('\nTotal test duration: %5.2fs\n',sum(table(results).Duration))
 
-%% display results
 if all([results.Passed])
   fprintf('\nPassed all tests\n')
   passed = true;
-else
-  fprintf('\nSome tests Failed or Incomplete\n')
-  if any([results.Incomplete])
-    fprintf('\nIncomplete:\n')
-    disp(table(results([results.Incomplete])))
-  end
-  if any([results.Failed])
-    fprintf('\nFailed:\n')
-    disp(table(results([results.Failed])));
-  end
+elseif any([results.Failed])
+  fprintf('\nSome tests Failed\n')
+  disp(table(results([results.Failed])))
   passed = false;
+elseif any([results.Incomplete])
+  fprintf('\nSome tests Incomplete\n')
+  disp(table(results([results.Incomplete])));
+  passed = true; % pass tests even if some were skipped
+else
+  % the conditions above should cover all cases, otherwise 
+  error('something is very wrong - please check')
+end
+
 end

matlab/setpaths_gdat.m

+gdatpaths.m
\ No newline at end of file

matlab/tests/check_gdatpaths.m

+classdef check_gdatpaths < matlab.unittest.fixtures.Fixture
+  methods
+    function setup(fixture)
+      % caller
+      oldpwd = pwd;
+      testfolder = fileparts(mfilename('fullpath')); % relative path w.r.t. this file
+      cd(testfolder); % go into test folder so you might not see local gdat.m
+      gdatpath = fileparts(which('gdat'));
+      cd(oldpwd);
+      fixture.assertTrue(startsWith(testfolder,gdatpath),...
+          sprintf('test folder path does not contain path of gdat: %s\n This file is in: %s\n which(''gdat''):\n Please run setpaths_gdat() to set the paths',...
+          testfolder,which('gdat')));
+    end
+  end
+end

matlab/tests/check_mds.m

 classdef check_mds < matlab.unittest.fixtures.Fixture
   methods
     function setup(fixture)
-      fixture.assumeFalse(~exist('mdsdata','file'),'mdsdata not found - is mds on path?');
+      fixture.assumeFalse(~exist('mdsconnect','file'),'mdsconnect not found - is mds on path?');
       % other environment checks here
     end
   end
-end
\ No newline at end of file
+end

matlab/tests/get_all_gdat_requests.m

@@ -5,12 +5,17 @@ function requests = get_all_gdat_requests(machine,testcase)
 switch machine
   % list of calls that take some time, to be skipped for fast tests
   case 'TCV'
-      slowlist = {'rtc','mhd','mpx','sxr','psi'};
+      slowlist = {'rtc','mpx','sxr','psi'};
+      excludelist = {'scd'}; % cases to skip for good reason
   case 'AUG'
       slowlist = {'sxr','cxrs','transp','te_rho','ne_rho','nete_rho',...
-        'ece_rho','eced_rho','cxrs_rho','eqdsk','equil'}; 
+        'ece_rho','eced_rho','cxrs_rho','eqdsk','equil'};
+      excludelist = {};
 end
 
+% filter requests
+requests = requests(~ismember(requests,excludelist));
+
 switch testcase
   case 'fast'
     requests = requests(~ismember(requests,slowlist));

matlab/tests/setup_gdatpaths.m

-classdef setup_gdatpaths < matlab.unittest.fixtures.Fixture
-  methods
-    function setup(fixture)
-      p = path; % get current matlab path (pre-testing)
-      fixture.addTeardown(@path,p); % teardown function to restore path
-      gdatpaths; % script to set up paths
-    end
-  end
-end
\ No newline at end of file

matlab/tests/test_requestnames.m

 classdef (SharedTestFixtures={...
-    check_mds,setup_gdatpaths}) ...
+    check_mds,check_gdatpaths}) ...
     test_requestnames < matlab.unittest.TestCase
-
+  
   properties (Abstract)
     Machine;
   end
-
+  
   properties(TestParameter,Abstract)
     % parameters that will vary during tests
     shot;
     requests_fast; % placeholders
     requests_slow;
   end
-
-  methods(Test,TestTags = {'fast'})
-    function test_gdat_call_fast(testCase,shot,requests_fast)
-      test_gdat_call(testCase,shot,requests_fast);
+  
+  methods(Static)
+    function test_gdat_call(testCase,shot,request)
+      % actual function to test gdat call
+      testCase.assertTrue(isnumeric(str2double(shot)));
+      testCase.assertTrue(ischar(request));
+      
+      % gdat call
+      gdat_call = sprintf(['gdat_' lower(testCase.Machine) '(%s,''%s'')'],shot,request);
+      do_gdat_call = 1;
+      
+      switch request
+        case 'eqdsk'
+          % avoid writing files in /tmp, may not be allowed
+          gdat_call = sprintf(['gdat_%s(%s,''%s'',''write'',0)'],lower(testCase.Machine),shot,request);
+      end
+      
+      % logging
+      fprintf('Testing gdat call: %s\n',gdat_call);
+      
+      if do_gdat_call
+        gdat_out = eval(gdat_call); %#ok<NASGU>
+      else
+        gdat_out = struct([]);
+      end
+      
+      % in some future: check for warnings
+      %gdat_out = verifyWarningFree(testCase,eval(gdat_call),...
+      %  'Warning issued from gdat call:\n   %s\n',gdat_call);
+      
+      % (add optional sanity checks of gdat_out here)
     end
   end
-
-  methods(Test,TestTags = {'slow'})
-    function test_gdat_call_slow(testCase,shot,requests_slow)
-      test_gdat_call(testCase,shot,requests_slow);
-    end
-  end
-
-end
-
-function test_gdat_call(testCase,shot,request)
-% actual function to test gdat call
-testCase.assertTrue(isnumeric(str2double(shot)));
-testCase.assertTrue(ischar(request));
-
-% gdat call
-gdat_call = sprintf(['gdat_' lower(testCase.Machine) '(%s,''%s'')'],shot,request);
-do_gdat_call = 1;
-
-switch request
- case 'eqdsk'
-  % avoid writing files in /tmp, may not be allowed
-  gdat_call = sprintf(['gdat_' lower(testCase.Machine) '(%s,''%s'',''write'',0)'],shot,request);
- case 'scd'
-  % scd dummy call with error information in gdat, do not do test (done with rtc)
-  do_gdat_call = 0;
- case 'rtc'
-  if exist('erase') ~= 2
-    do_gdat_call = 0; % get_scd_mems works with newer versions which contains erase function
-  end
-end
-
-% logging
-fprintf('Testing gdat call: %s\n',gdat_call);
-
-if do_gdat_call
-  gdat_out = eval(gdat_call); %#ok<NASGU>
-else
-  gdat_out = struct([]);
-end
-
-% in some future: check for warnings
-%gdat_out = verifyWarningFree(testCase,eval(gdat_call),...
-%  'Warning issued from gdat call:\n   %s\n',gdat_call);
-
-% (add optional sanity checks of gdat_out here)
-end
+end
\ No newline at end of file

matlab/tests/test_requestnames_aug.m

-classdef test_requestnames_aug < test_requestnames
+classdef (TestTags={'aug'})test_requestnames_aug < test_requestnames
   % everything implemented in superclass
   
   properties
@@ -12,4 +12,15 @@ classdef test_requestnames_aug < test_requestnames
     requests_slow = get_all_gdat_requests('AUG','slow');  
   end
   
+  methods(Test,TestTags = {'fast'})
+    function test_gdat_call_fast(testCase,shot,requests_fast)
+      testCase.test_gdat_call(testCase,shot,requests_fast);
+    end
+  end
+  
+  methods(Test,TestTags = {'slow'})
+    function test_gdat_call_slow(testCase,shot,requests_slow)
+      testCase.test_gdat_call(testCase,shot,requests_slow);
+    end
+  end
 end
\ No newline at end of file

matlab/tests/test_requestnames_tcv.m

-classdef test_requestnames_tcv < test_requestnames
+classdef (TestTags={'tcv'})test_requestnames_tcv < test_requestnames
   % everything is implemented in superclass!
 
   properties
@@ -11,5 +11,17 @@ classdef test_requestnames_tcv < test_requestnames
     requests_fast = get_all_gdat_requests('TCV','fast');
     requests_slow = get_all_gdat_requests('TCV','slow');
   end
-
+  
+  methods(Test,TestTags = {'fast'})
+    function test_gdat_call_fast(testCase,shot,requests_fast)
+      testCase.test_gdat_call(testCase,shot,requests_fast);
+    end
+  end
+  
+  methods(Test,TestTags = {'slow'})
+    function test_gdat_call_slow(testCase,shot,requests_slow)
+      testCase.test_gdat_call(testCase,shot,requests_slow);
+    end
+  end
+  
 end

matlab/TCV_IMAS/test_tcv_get_ids.m → matlab/tests/test_tcv_get_ids.m

-classdef (SharedTestFixtures={...
-    check_mds,setup_gdatpaths})...
+classdef (TestTags={'tcv_imas'},SharedTestFixtures={...
+    check_mds,check_gdatpaths})...
     test_tcv_get_ids < matlab.unittest.TestCase
+    % Tests link between TCV data to matlab ids structure and  ids empty structure
+    % The link between ids data to saving on IMAS database, or reading from an IMAS database should be performed
+    %   in another function like test_tcv_put_ids with only 'imas' as TestTags, performed where IMAS is installed
 
   properties(TestParameter)
     shot = {-1,40000,61400};
     ids_name = {'pf_active','wall'};
   end
 
-  methods(Test,TestTags = {'imas'})
+  methods(Test)
     function test_get_ids_list(testCase,shot)
       gg = testCase.assertWarning(@() gdat(shot,'ids'),'gdat:EmptyIDSName');
 

matlab/tests_matlab.m

 function tests_matlab(test_case)
-%
-% function to call tests from test_script.sh
-% calls passed = run_gdat_tests(test_case);
-% and exits matlab
-%
 try
-   if verLessThan('matlab','8.5.0')
-    disp('gdat tests do not work for matlab version <8.5.0, skipping tests');
-    passed = true;
-   else
-    fprintf('\n Running test file: %s\n',mfilename('fullpath'));
-    fprintf('     Time: %s\n',datestr(now));
-    passed = run_gdat_tests(test_case); % add a call to your test script here, with optional test_case input
-   end
+   fprintf('\n Running test file: %s\n',mfilename('fullpath'));
+   fprintf('     Time: %s\n',datestr(now));
+   
+   passed = run_gdat_tests(test_case); % call to your test script here, with optional test_case input
    exit_code = int32(~passed); % convert to bash shell convention
 catch ME
    disp(getReport(ME))