data.hpp

Go to the documentation of this file.

/*
  DATA.hpp  -  read and write a table with CSV data

  Copyright (C)         Lumiera.org
    2022,               Hermann Vosseler <Ichthyostega@web.de>

  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License as
  published by the Free Software Foundation; either version 2 of
  the License, or (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

*/


#ifndef LIB_STAT_DATA_H
#define LIB_STAT_DATA_H


#include "lib/error.hpp"
#include "lib/nocopy.hpp"
#include "lib/stat/csv.hpp"
#include "lib/stat/file.hpp"
#include "lib/format-string.hpp"
#include "lib/util.hpp"

#include <type_traits>
#include <utility>
#include <fstream>
#include <vector>
#include <string>
#include <limits>
#include <deque>


namespace lib {
namespace stat{
  
  namespace error = lumiera::error;
  
  using std::move;
  using std::tuple;
  using std::vector;
  using std::string;
  using util::isnil;
  using util::unConst;
  using util::_Fmt;
  using util::min;
  
  
  
  
  template<typename VAL>
  struct Column
    : util::MoveOnly
    {
      string header;
      vector<VAL> data;
      
      using ValueType = VAL;
      
      
      Column (string headerID)
        : header{headerID}
        , data{}
        { }
      
      
      VAL&
      get()
        {
          if (isnil (data))
              throw error::State{"No rows in DataTable yet"};
          return data.back();
        }
      
      operator VAL&()
        {
          return get();
        }
      
      operator VAL const&()  const
        {
          return unConst(this)->get();
        }
      
      template<typename X>
      VAL& operator= (X&& newVal)
      {
          return get() = std::forward<X> (newVal);
      }
    };
  
  
  
  
  
  /******************************************************************************************/
  template<class TAB>
  class DataTable
      : public TAB
      , util::MoveOnly
    {
      fs::path filename_;
      
    public:
      DataTable(fs::path csvFile ="")
        : filename_{fs::consolidated (csvFile)}
        {
          loadData();
        }
      
      DataTable (CSVData const& csv)
        : filename_{}
        {
          appendFrom (csv);
        }
      
      
      /* === Data Access === */
      
      static constexpr size_t columnCnt = std::tuple_size_v<decltype(std::declval<TAB>().allColumns())>;
      
      bool
      empty()  const
        {
          return 0 == this->size();
        }
      
      size_t
      size()  const
        {
          if (0 == columnCnt) return 0;
          size_t rowCnt = std::numeric_limits<size_t>::max();
          forAllColumns(
                    [&](auto& col)
                      {
                        rowCnt = min (rowCnt, col.data.size());
                      }); // the smallest number of data points found in any column
          return rowCnt;
        }
      
      CSVData
      renderCSV()  const
        {
          CSVData csv{{}};
          csv.reserve (size()+1);
          auto header = generateHeaderSpec();
          std::swap (csv[0], header);
          for (uint i=0; i < size(); ++i)
            csv.emplace_back (formatCSVRow(i));
          return csv;
        }
      
      
      
      /* === Manipulation === */
      
      void
      newRow()
        {
            forAllColumns(
                    [siz = size()+1]
                    (auto& col)
                      {
                        col.data.resize (siz);
                      });
        }
      
      void
      dupRow()
        {
          if (empty())
            newRow();
          else
            forAllColumns(
                    [](auto& col)
                      {
                        col.data.emplace_back (col.data.back());
                      });
        }
      
      void
      dropLastRow()
        {
          if (not empty())
            forAllColumns(
                    [](auto& col)
                      {
                        size_t siz = col.data.size();
                        col.data.resize (siz>0? siz-1 : 0);
                      });
        }
      
      void
      reserve (size_t expectedCapacity)
        {
            forAllColumns(
                    [=](auto& col)
                      {
                        col.data.reserve(expectedCapacity);
                      });
        }
      
      void
      clear()
        {
            forAllColumns(
                    [](auto& col)
                      {
                        col.data.clear();
                      });
        }
      
      void
      appendFrom (CSVData const& csv)
        {
          if (isnil (csv)) return;
          verifyHeaderSpec (csv[0]);
          for (size_t row=1; row<csv.size(); ++row)
            if (not isnil (csv[row]))
              appendRowFromCSV (csv[row]);
        }
      
      
      
      void
      save (size_t lineLimit =std::numeric_limits<size_t>::max()
           ,bool backupOld =false)
        {
          if (filename_.empty())
            throw error::Logic{"Unable to save DataFile without filename given."};
          
          fs::path newFilename{filename_};
          newFilename += ".tmp";
          
          std::ofstream csvFile{newFilename, std::ios_base::out | std::ios_base::trunc};
          if (not csvFile.good())
            throw error::State{_Fmt{"Unable to create CSV output file %s"}
                                   % newFilename};
          saveData (csvFile, lineLimit);
          
          if (backupOld)
            {
              fs::path oldFile{filename_};
              oldFile += ".bak";
              if (fs::exists (filename_))
                  fs::rename (filename_, oldFile);
            }
          fs::rename (newFilename, filename_);
          filename_ = fs::consolidated(filename_);
        }                // lock onto absolute path
      
      
      void
      saveAs (fs::path newStorage
             ,size_t lineLimit =std::numeric_limits<size_t>::max())
        {
          newStorage = fs::consolidated (newStorage);
          if (fs::exists(newStorage))
            throw error::Invalid{_Fmt{"Storing DataFile rejected: target %s exists already"}
                                     % newStorage};
          if (not (newStorage.parent_path().empty()
                   or fs::exists(newStorage.parent_path())))
            throw error::Invalid{_Fmt{"DataFile(%s) placed into nonexistent directory %s"}
                                     % newStorage.filename() % newStorage.parent_path()};
          filename_ = newStorage;
          save (lineLimit);
        }
      
      
    private: /* === Implementation === */
      
      template<class OP>
      void
      forAllColumns (OP&& doIt)  const
        {
          lib::meta::forEach (unConst(this)->allColumns()
                             ,std::forward<OP> (doIt));
        }
      
      void
      loadData()
        {
          if (not (filename_.parent_path().empty()
                   or fs::exists(filename_.parent_path())))
            throw error::Invalid{_Fmt{"DataFile(%s) placed into nonexistent directory %s"}
                                     % filename_.filename() % filename_.parent_path()};
          if (not fs::exists(filename_))
            return; // leave the table empty
          
          std::ifstream csvFile{filename_};
          if (not csvFile.good())
            throw error::Config{_Fmt{"unable to read CSV data file %s"} % filename_};
          
          std::deque<string> rawLines;
          for (string line; std::getline(csvFile, line); )
            rawLines.emplace_back (move(line));
          
          if (rawLines.size() < 1) return;
          verifyHeaderSpec (rawLines[0]);
          
          // we know the number of rows now...
          reserve (rawLines.size() - 1);
          
          // storage in file is backwards, with newest data on top
          for (size_t row = rawLines.size()-1; 0<row; --row)
            if (not isnil(rawLines[row]))
              appendRowFromCSV (rawLines[row]);
        }
      
      
      void
      saveData (std::ofstream& csvFile, size_t lineLimit)
        {
          csvFile << generateHeaderSpec() << "\n";
          if (empty())
            return;
          lineLimit = size() > lineLimit? size()-lineLimit : 0;
          // store newest data first, possibly discard old data
          for (size_t row = size(); lineLimit < row; --row)
            csvFile << formatCSVRow(row-1) << "\n";
        }
      
      
      void
      verifyHeaderSpec (string headerLine)
        {
          CsvParser header{headerLine};
          forAllColumns(
                  [&](auto& col)
                    {
                      if (*header != col.header)
                        throw error::Invalid{_Fmt{"Header mismatch in CSV file %s. "
                                                  "Expecting column(%s) but found \"%s\""}
                                                 % filename_ % col.header % *header};
                      ++header;
                    });
        }
      
      CSVLine
      generateHeaderSpec()  const
        {
          CSVLine csv;
          forAllColumns(
                  [&](auto& col)
                    {
                      csv += col.header;
                    });
          return csv;
        }
      
      
      void
      appendRowFromCSV (string line)
        {
          newRow();
          CsvParser csv(line);
          forAllColumns(
                  [&](auto& col)
                    {
                      if (not csv)
                        {
                          if (csv.isParseFail())
                            csv.fail();
                          else
                            throw error::Invalid{_Fmt{"Insufficient data; only %d fields, %d expected. Line:%s"}
                                                     % csv.getParsedFieldCnt() % columnCnt % line};
                        }
                      
                      using Value = typename std::remove_reference<decltype(col)>::type::ValueType;
                      col.get() = parseAs<Value>(*csv);
                      ++csv;
                    });
          if (csv)
            throw error::Invalid{_Fmt{"Excess data fields in CSV. Expect %d fields. Line:%s"}
                                     % columnCnt % line};
        }
      
      
      CSVLine
      formatCSVRow (size_t rownum)  const
        {
          if (this->empty())
            throw error::Logic{"Attempt to access data from empty DataTable."};
          if (rownum >= this->size())
            throw error::Logic{_Fmt{"Attempt to access row #%d beyond range [0..%d]."}
                                   % rownum % (size()-1)};
          
          CSVLine csvLine;
          forAllColumns(
                  [&](auto& col)
                    {
                      csvLine += col.data.at(rownum);
                    });
          return csvLine;
        }
    };
  
}} // namespace lib::stat
#endif /*LIB_STAT_DATA_H*/