Scid  4.6.5
scidbase.h
Go to the documentation of this file.
1 /*
2 # Copyright (C) 2014-2016 Fulvio Benini
3 
4 * This file is part of Scid (Shane's Chess Information Database).
5 *
6 * Scid is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 *
10 * Scid is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with Scid. If not, see <http://www.gnu.org/licenses/>.
17 */
18 
19 #ifndef SCIDBASE_H
20 #define SCIDBASE_H
21 
22 #include "bytebuf.h"
23 #include "codec.h"
24 #include "fastgame.h"
25 #include "game.h"
26 #include "index.h"
27 #include "namebase.h"
28 #include "tree.h"
29 #include "undoredo.h"
30 #include <vector>
31 
32 class SortCache;
33 
34 
35 const gamenumT INVALID_GAMEID = 0xffffffff;
36 
37 struct scidBaseT {
38  struct Stats {
39  uint flagCount[IndexEntry::IDX_NUM_FLAGS]; // Num of games with each flag set.
42  uint64_t nYears;
43  uint64_t sumYears;
46  uint64_t sumRatings;
49 
50  Stats(const scidBaseT* dbase);
51 
52  struct Eco {
55 
56  Eco();
57  };
58  const Eco* getEcoStats(const char* ecoStr) const;
59 
60  private:
61  Eco ecoEmpty_;
62  Eco ecoValid_;
63  Eco ecoStats_ [(1 + (1<<16)/131)*27];
64  Eco ecoGroup1_[(1 + (1<<16)/131)/100];
65  Eco ecoGroup2_[(1 + (1<<16)/131)/10];
66  Eco ecoGroup3_[(1 + (1<<16)/131)];
67  };
68 
69  struct TreeStat {
71  std::string SAN;
72  int resultW, resultD, resultB;
73  double exp;
74  int ngames, nexp;
75 
76  public:
77  TreeStat();
78  void add(int result, int eloW, int eloB);
79  bool operator<(const TreeStat& cmp) const { return ngames > cmp.ngames; }
80 
81  private:
82  static double expVect_[1600];
83  };
84 
85  scidBaseT();
86  ~scidBaseT();
87 
89  fileModeT mode,
90  const char* filename = 0,
91  const Progress& progress = Progress());
92 
93  errorT Close ();
94 
95  const std::string& getFileName() const { return fileName_; }
96  bool isReadOnly() const { return fileMode_ == FMODE_ReadOnly; }
97  gamenumT numGames() const { return idx->GetNumGames(); }
98  errorT getExtraInfo(const std::string& tagname, std::string* res) const;
99  errorT setExtraInfo(const std::string& tagname, const char* new_value);
100 
102  return idx->GetEntry(g);
103  }
105  if (g < numGames()) return getIndexEntry(g);
106  return 0;
107  }
108  const NameBase* getNameBase() const {
109  return nb;
110  }
111  FastGame getGame(const IndexEntry* ie) const {
112  uint length = ie->GetLength();
113  const byte* b = codec_->getGameData(ie->GetOffset(), length);
114  if (b == 0) length = 0; // Error
115  return FastGame::Create(b, b + length);
116  }
117  errorT getGame(const IndexEntry* ie, ByteBuffer* destBuf) const {
118  uint length = ie->GetLength();
119  const byte* b = codec_->getGameData(ie->GetOffset(), length);
120  if (b == 0) return ERROR_FileRead;
121  // The data for the game is not actually copied into the bytebuffer, which would
122  // be slower and a waste of time if the bytebuffer is not going to be modified.
123  destBuf->ProvideExternal(const_cast<byte*>(b), length);
124  return OK;
125  }
126 
127  struct GamePos {
128  unsigned int RAVdepth;
129  unsigned int RAVnum;
130  std::string FEN;
131  std::vector<int> NAGs;
132  std::string comment;
133  std::string lastMoveSAN;
134  };
135  errorT getGame(const IndexEntry* ie, std::vector<GamePos>& dest);
136 
137  errorT importGame(const scidBaseT* srcBase, uint gNum);
138  errorT importGames(const scidBaseT* srcBase, const HFilter& filter, const Progress& progress);
139  template <class T, class P>
140  errorT importGames(T& codec, const P& progress, uint& nImported, std::string& errorMsg);
141 
142  /**
143  * Add or replace a game into the database.
144  * @param game: valid pointer to a Game object with the data of the game.
145  * @param replacedgameId: id of the game to replace.
146  * If >= numGames(), a new game will be added.
147  * @returns OK if successful or an error code.
148  */
149  errorT saveGame(Game* game, gamenumT replacedGameId = INVALID_GAMEID);
150  // TODO: private:
152 
153 
154  bool getFlag(uint flag, uint gNum) const {
155  return idx->GetEntry(gNum)->GetFlag (flag);
156  }
157  errorT setFlag(bool value, uint flag, uint gNum);
158  errorT setFlag(bool value, uint flag, const HFilter& filter);
159  errorT invertFlag(uint flag, uint gNum);
160  errorT invertFlag(uint flag, const HFilter& filter);
161 
162  /**
163  * Filters
164  * @filterId: unique identifier of a Filter
165  *
166  * A Filter is a selection of games, usually obtained searching the
167  * database. A new Filter is created calling the function newFilter()
168  * and must be released calling the function deleteFilter().
169  * A composed Filter is a special contruct created combining two Filters
170  * and includes only the games contained in both Filters.
171  * A composed Filter should NOT be released.
172  */
173  std::string newFilter();
174  std::string composeFilter(const std::string& mainFilter,
175  const std::string& maskFilter) const;
176  void deleteFilter(const char* filterId);
177  HFilter getFilter(const std::string& filterId) const {
178  return getFilterHelper(filterId, false);
179  }
180  HFilter getMainFilter(const std::string& filterId) const {
181  return getFilterHelper(filterId, true);
182  }
183 
184  const Stats& getStats() const;
185  std::vector<scidBaseT::TreeStat> getTreeStat(const HFilter& filter);
187  if (nameFreq_[nt].size() == 0) idx->calcNameFreq(*nb, nameFreq_);
188  return nameFreq_[nt][id];
189  }
190 
191  errorT getCompactStat(uint* n_deleted,
192  uint* n_unused,
193  uint* n_sparse,
194  uint* n_badNameId);
195  errorT compact(const Progress& progress);
196 
197 
198  /**
199  * Increment the reference count of a SortCache object matching @e criteria.
200  * @param criteria: the list of fields by which games will be ordered.
201  * Each field should be followed by '+' to indicate an
202  * ascending order or by '-' for a descending order.
203  * @returns a pointer to a SortCache object in case of success, NULL
204  * otherwise.
205  */
206  SortCache* createSortCache(const char* criteria);
207 
208  /**
209  * Decrement the reference count of the SortCache object matching @e
210  * criteria. Cached objects with refCount <= 0 are destroyed independently
211  * from the value of @e criteria.
212  * @param criteria: the list of fields by which games will be ordered.
213  * Each field should be followed by '+' to indicate an
214  * ascending order or by '-' for a descending order.
215  */
216  void releaseSortCache(const char* criteria);
217 
218  /**
219  * Retrieve a list of ordered game indexes sorted by @e criteria.
220  * This function will be much faster if a SortCache object matching @e
221  * criteria already exists (previously created with @e createSortCache).
222  * @param criteria: the list of fields by which games will be ordered.
223  * Each field should be followed by '+' to indicate an
224  * ascending order or by '-' for a descending order.
225  * @param start: the offset of the first row to return.
226  * The offset of the initial row is 0.
227  * @param count: maximum number of rows to return.
228  * @param filter: a reference to a valid (!= NULL) HFilter object.
229  * Games not included into the filter will be ignored.
230  * @param[out] destCont: valid pointer to an array where the sorted list of
231  * games will be stored (should be able to contain at
232  * least @e count elements).
233  * @returns the number of games' ids stored into @e destCont.
234  */
235  size_t listGames(const char* criteria, size_t start, size_t count,
236  const HFilter& filter, gamenumT* destCont);
237 
238  /**
239  * Get the sorted position of a game.
240  * This function will be much faster if a SortCache object matching @e
241  * criteria already exists (previously created with @e createSortCache).
242  * @param criteria: the list of fields by which games will be ordered.
243  * Each field should be followed by '+' to indicate an
244  * ascending order or by '-' for a descending order.
245  * @param filter: a reference to a valid (!= NULL) HFilter object.
246  * Games not included into the filter will be ignored.
247  * @param gameId: the id of the game.
248  * @returns the sorted position of @e gameId.
249  */
250  size_t sortedPosition(const char* criteria, const HFilter& filter,
251  gamenumT gameId);
252 
253  void setDuplicates(uint* duplicates) {
254  if (duplicates_ != NULL) { delete[] duplicates_; duplicates_ = NULL; }
255  duplicates_ = duplicates;
256  }
258  return (duplicates_ == NULL) ? 0 : duplicates_[gNum];
259  }
260 
261  // TODO: private:
262  /**
263  * This function must be called before modifying the games of the database.
264  * Currently this function do not guarantees that the database is not altered
265  * in case of errors.
266  */
267  void beginTransaction();
268 
269  // TODO: private:
270  /**
271  * Update caches and flush the database's files.
272  * This function must be called after changing one or more games.
273  * @param gameId: id of the modified game
274  * INVALID_GAMEID to update all games.
275  * @returns OK if successful or an error code.
276  */
278 
279 public:
280  Index* idx; // the Index file in memory for this base.
281  NameBase* nb; // the NameBase file in memory.
282  bool inUse; // true if the database is open (in use).
288 
289 
290  //TODO: this vars do not belong to scidBaseT class
291  Game* game; // the active game for this base.
292  int gameNumber; // game number of active game.
293  bool gameAltered; // true if game is modified
295 
296 private:
297  ICodecDatabase* codec_;
298  std::string fileName_; // File name without ".si" suffix
299  fileModeT fileMode_; // Read-only, write-only, or both.
300  std::vector< std::pair<std::string, Filter*> > filters_;
301  mutable Stats* stats_;
302  std::vector <int> nameFreq_ [NUM_NAME_TYPES];
303  uint* duplicates_; // For each game: idx of duplicate game + 1 (0 if there is no duplicate).
304  std::vector< std::pair<std::string, SortCache*> > sortCaches_;
305 
306 private:
307  scidBaseT(const scidBaseT&);
308  scidBaseT& operator=(const scidBaseT&);
309  void clear();
310  GamePos makeGamePos(Game& game, unsigned int ravNum);
311  errorT importGameHelper(const scidBaseT* sourceBase, uint gNum);
312 
313  void extendFilters();
314  Filter* fetchFilter(const std::string& filterId) const;
315  HFilter getFilterHelper(const std::string& filterId,
316  bool unmasked = false) const;
317  SortCache* getSortCache(const char* criteria);
318 };
319 
320 inline void scidBaseT::TreeStat::add(int result, int eloW, int eloB) {
321  ngames++;
322  double r = 0;
323  switch (result) {
324  case RESULT_White: resultW++; r = 1; break;
325  case RESULT_Draw: resultD++; r = 0.5; break;
326  case RESULT_Black: resultB++; break;
327  default: return;
328  }
329  if (eloW == 0 || eloB == 0) return;
330  int eloDiff = eloB - eloW;
331  if (eloDiff < 800 && eloDiff >= -800) {
332  exp += r - expVect_[eloDiff+800];
333  nexp++;
334  }
335 }
336 
337 template <class T, class P>
338 inline errorT scidBaseT::importGames(T& codec, const P& progress, uint& nImported, std::string& errorMsg) {
340  errorT res;
341  Game g;
342  nImported = 0;
343  while ((res = codec.parseNext(&g)) != ERROR_NotFound) {
344  if (res != OK) continue;
345 
346  res = saveGameHelper(&g, INVALID_GAMEID);
347  if (res != OK) break;
348 
349  if ((++nImported % 200) == 0) {
350  std::pair<size_t, size_t> count = codec.parseProgress();
351  if (!progress.report(count.first, count.second)) {
352  res = ERROR_UserCancel;
353  break;
354  }
355  }
356  }
357 
358  errorMsg = codec.parseErrors();
359  endTransaction();
360  progress.report(1,1);
361 
362  if (res == ERROR_NotFound) res = OK;
363  return res;
364 }
365 
366 inline errorT scidBaseT::invertFlag(uint flag, uint gNum) {
367  return setFlag(! getFlag(flag, gNum), flag, gNum);
368 }
369 
370 inline errorT scidBaseT::invertFlag(uint flag, const HFilter& filter) {
371  errorT res = OK;
372  for (gamenumT i = 0, n = numGames(); i < n; i++) {
373  if (filter != 0 && filter->get(i) == 0) continue;
374  res = invertFlag(flag, i);
375  if (res != OK) return res;
376  }
377  return res;
378 }
379 
380 inline errorT scidBaseT::setFlag(bool value, uint flag, uint gNum){
381  ASSERT(gNum < idx->GetNumGames());
382  IndexEntry* ie = idx->FetchEntry (gNum);
383  ie->SetFlag (flag, value);
384  errorT res = idx->WriteEntry (ie, gNum, false);
385  if (stats_ != NULL) { delete stats_; stats_ = NULL;}
386  // TODO: necessary only for sortcaches with SORTING_deleted (and SORTING_flags when implemented)
387  // idx->IndexUpdated(gNum);
388  return res;
389 }
390 
391 inline errorT scidBaseT::setFlag(bool value, uint flag, const HFilter& filter) {
392  errorT res = OK;
393  for (gamenumT gNum = 0, n = numGames(); gNum < n; gNum++) {
394  if (filter != 0 && filter->get(gNum) == 0) continue;
395  res = setFlag(value, flag, gNum);
396  if (res != OK) return res;
397  }
398  return res;
399 }
400 
401 
402 
403 #endif
404 
bool getFlag(uint flag, uint gNum) const
Definition: scidbase.h:154
unsigned char byte
Definition: common.h:97
void deleteFilter(const char *filterId)
Definition: scidbase.cpp:448
bool GetFlag(uint32_t mask) const
Definition: indexentry.h:304
errorT getGame(const IndexEntry *ie, ByteBuffer *destBuf) const
Definition: scidbase.h:117
uint32_t GetLength() const
Definition: indexentry.h:89
addwin args
Definition: board.tcl:994
Definition: indexentry.h:400
uint idNumberT
Definition: namebase.h:39
const errorT OK
Definition: error.h:23
const uint NUM_RESULT_TYPES
Definition: common.h:182
UndoRedo< Game, 100 > gameAlterations
Definition: scidbase.h:294
unsigned int RAVdepth
Definition: scidbase.h:128
SortCache * createSortCache(const char *criteria)
Increment the reference count of a SortCache object matching criteria.
Definition: scidbase.cpp:855
uint dateT
Definition: common.h:155
#define ASSERT(f)
Definition: common.h:67
gamenumT numGames() const
Definition: scidbase.h:97
uint getNameFreq(nameT nt, idNumberT id)
Definition: scidbase.h:186
const resultT RESULT_Black
Definition: common.h:187
const IndexEntry * getIndexEntry(gamenumT g) const
Definition: scidbase.h:101
Definition: misc.h:124
NameBase * nb
Definition: scidbase.h:281
errorT Close()
Definition: scidbase.cpp:124
uint64_t sumYears
Definition: scidbase.h:43
Defines the ICodecDatabase interface, which encapsulates the data representation of databases...
TreeCache treeCache
Definition: scidbase.h:284
unsigned int RAVnum
Definition: scidbase.h:129
errorT saveGame(Game *game, gamenumT replacedGameId=INVALID_GAMEID)
Add or replace a game into the database.
Definition: scidbase.cpp:330
errorT endTransaction(gamenumT gameId=INVALID_GAMEID)
Update caches and flush the database&#39;s files.
Definition: scidbase.cpp:169
const gamenumT INVALID_GAMEID
Definition: scidbase.h:35
Index * idx
Definition: scidbase.h:280
void setDuplicates(uint *duplicates)
Definition: scidbase.h:253
std::string SAN
Definition: scidbase.h:71
uint results[NUM_RESULT_TYPES]
Definition: scidbase.h:54
void add(int result, int eloW, int eloB)
Definition: scidbase.h:320
const resultT RESULT_Draw
Definition: common.h:188
bool inUse
Definition: scidbase.h:282
std::string comment
Definition: scidbase.h:132
errorT importGame(const scidBaseT *srcBase, uint gNum)
Definition: scidbase.cpp:350
void SetFlag(uint32_t flagMask, bool b)
Definition: indexentry.h:334
void beginTransaction()
This function must be called before modifying the games of the database.
Definition: scidbase.cpp:163
static FastGame Create(const byte *v_begin, const byte *v_end)
Definition: fastgame.h:245
errorT setExtraInfo(const std::string &tagname, const char *new_value)
Definition: scidbase.cpp:199
const std::string & getFileName() const
Definition: scidbase.h:95
std::string newFilter()
Filters : unique identifier of a Filter.
Definition: scidbase.cpp:414
Filter * dbFilter
Definition: scidbase.h:286
const Stats & getStats() const
Statistics.
Definition: scidbase.cpp:497
uint64_t sumRatings
Definition: scidbase.h:46
const errorT ERROR_FileRead
Definition: error.h:33
uint nameT
Definition: namebase.h:29
errorT invertFlag(uint flag, uint gNum)
Definition: scidbase.h:366
const errorT ERROR_UserCancel
Definition: error.h:27
bool operator<(const TreeStat &cmp) const
Definition: scidbase.h:79
Game * game
Definition: scidbase.h:291
sizew
Definition: board.tcl:619
bool isReadOnly() const
Definition: scidbase.h:96
~scidBaseT()
Definition: scidbase.cpp:75
This interface separates the logic of a database from its representation.
Definition: codec.h:43
size_t sortedPosition(const char *criteria, const HFilter &filter, gamenumT gameId)
Get the sorted position of a game.
Definition: scidbase.cpp:872
const resultT RESULT_White
Definition: common.h:186
uint32_t uint
Definition: common.h:99
errorT saveGameHelper(Game *game, gamenumT gameId)
Definition: scidbase.cpp:337
bool gameAltered
Definition: scidbase.h:293
IndexEntry * FetchEntry(gamenumT g)
FetchEntry() - return a modifiable pointer to a game&#39;s IndexEntry.
Definition: index.h:209
Stats(const scidBaseT *dbase)
Definition: scidbase.cpp:507
Definition: index.h:61
gamenumT GetNumGames() const
Header getter functions.
Definition: index.h:141
uint32_t GetOffset() const
Definition: indexentry.h:87
errorT getCompactStat(uint *n_deleted, uint *n_unused, uint *n_sparse, uint *n_badNameId)
Definition: scidbase.cpp:652
errorT getExtraInfo(const std::string &tagname, std::string *res) const
Definition: scidbase.cpp:180
void releaseSortCache(const char *criteria)
Decrement the reference count of the SortCache object matching criteria.
Definition: scidbase.cpp:841
Filter * treeFilter
Definition: scidbase.h:287
std::vector< int > NAGs
Definition: scidbase.h:131
uint getDuplicates(gamenumT gNum)
Definition: scidbase.h:257
unsigned short errorT
Definition: error.h:20
void calcNameFreq(const NameBase &nb, std::vector< int >(&resVec)[NUM_NAME_TYPES]) const
calcNameFreq() - calculate the usage of NameBase&#39;s names : the NameBase linked to this Index : an arr...
Definition: index.h:122
const Eco * getEcoStats(const char *ecoStr) const
Definition: scidbase.cpp:581
uint nResults[NUM_RESULT_TYPES]
Definition: scidbase.h:44
Definition: tree.h:81
uint flagCount[IndexEntry::IDX_NUM_FLAGS]
Definition: scidbase.h:39
const IndexEntry * getIndexEntry_bounds(gamenumT g) const
Definition: scidbase.h:104
errorT importGames(const scidBaseT *srcBase, const HFilter &filter, const Progress &progress)
Definition: scidbase.cpp:361
HFilter getMainFilter(const std::string &filterId) const
Definition: scidbase.h:180
errorT compact(const Progress &progress)
Definition: scidbase.cpp:688
This class sorts games contained into an Index.
Definition: sortcache.h:50
Definition: game.h:227
ByteBuffer * bbuf
Definition: scidbase.h:285
int gameNumber
Definition: scidbase.h:292
std::vector< scidBaseT::TreeStat > getTreeStat(const HFilter &filter)
Definition: scidbase.cpp:619
void ProvideExternal(byte *data, uint length)
Definition: bytebuf.cpp:52
std::string lastMoveSAN
Definition: scidbase.h:133
byte get(gamenumT gnum) const
Definition: hfilter.h:168
treeT tree
Definition: scidbase.h:283
Definition: filter.h:32
byte colorT
Definition: common.h:112
const IndexEntry * GetEntry(gamenumT g) const
Definition: index.h:98
errorT Open(ICodecDatabase::Codec dbtype, fileModeT mode, const char *filename=0, const Progress &progress=Progress())
Definition: scidbase.cpp:89
uint gamenumT
Definition: common.h:159
fileModeT
Definition: common.h:144
errorT WriteEntry(const IndexEntry *ie, gamenumT idx, bool flush=true)
WriteEntry() - modify a game in the Index.
Definition: index.h:217
uint64_t nYears
Definition: scidbase.h:42
std::string FEN
Definition: scidbase.h:130
const errorT ERROR_NotFound
Definition: error.h:50
std::string composeFilter(const std::string &mainFilter, const std::string &maskFilter) const
Definition: scidbase.cpp:427
HFilter getFilter(const std::string &filterId) const
Definition: scidbase.h:177
const NameBase * getNameBase() const
Definition: scidbase.h:108
Definition: indexentry.h:54
FastGame getGame(const IndexEntry *ie) const
Definition: scidbase.h:111
size_t listGames(const char *criteria, size_t start, size_t count, const HFilter &filter, gamenumT *destCont)
Retrieve a list of ordered game indexes sorted by criteria.
Definition: scidbase.cpp:863
errorT setFlag(bool value, uint flag, uint gNum)
Definition: scidbase.h:380