Scid  4.6.5
scidbase.cpp
Go to the documentation of this file.
1 /*
2 * Copyright (C) 2014-2016 Fulvio Benini
3 
4 * This file is part of Scid (Shane's Chess Information Database).
5 *
6 * Scid is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 *
10 * Scid is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with Scid. If not, see <http://www.gnu.org/licenses/>.
17 */
18 
19 #include "scidbase.h"
20 #include "codec_memory.h"
21 #include "codec_pgn.h"
22 #include "codec_scid4.h"
23 #include "common.h"
24 #include "sortcache.h"
25 #include "stored.h"
26 #include <algorithm>
27 #include <math.h>
28 
30  fileModeT fMode, const char* filename,
31  const Progress& progress, Index* idx,
32  NameBase* nb) {
33  ICodecDatabase* res = 0;
34  errorT err = ERROR;
35  switch (codec) {
37  res = new CodecMemory();
38  break;
40  res = new CodecScid4();
41  break;
43  res = new CodecPgn();
44  break;
45  }
46 
47  if (res != 0) {
48  err = res->dyn_open(fMode, filename, progress, idx, nb);
49  if (err != OK && err != ERROR_NameDataLoss) {
50  delete res;
51  res = 0;
52  }
53  }
54  if (resError) *resError = err;
55  return res;
56 }
57 
59  idx = new Index;
60  nb = new NameBase;
61  game = new Game;
62  gameNumber = -1;
63  gameAltered = false;
64  inUse = false;
65  tree.moveCount = tree.totalCount = 0;
66  fileMode_ = FMODE_None;
67  codec_ = 0;
68  bbuf = new ByteBuffer(BBUF_SIZE);
69  dbFilter = new Filter(0);
70  treeFilter = new Filter(0);
71  duplicates_ = NULL;
72  stats_ = NULL;
73 }
74 
76  if (inUse)
77  Close();
78 
79  delete[] duplicates_;
80  delete idx;
81  delete nb;
82  delete game;
83  delete bbuf;
84  delete stats_;
85  delete dbFilter;
86  delete treeFilter;
87 }
88 
90  const char* filename, const Progress& progress) {
91  if (inUse) return ERROR_FileInUse;
92  if (filename == 0) filename = "";
93 
94  inUse = true;
95 
96  delete codec_;
97  errorT err = OK;
98  codec_ = ICodecDatabase::make(dbtype, &err, fMode, filename, progress, idx, nb);
99  if (codec_ == 0) {
100  idx->Close();
101  nb->Clear();
102  inUse = false;
103  return err;
104  }
105 
106  if (fMode == FMODE_Create) fMode = FMODE_Both;
107  fileMode_ = fMode;
108  fileName_ = filename;
109  gameNumber = -1;
110 
111  // Initialize the filters: all the games are included by default.
112  dbFilter->Init(numGames());
113  treeFilter->Init(numGames());
114  ASSERT(filters_.empty());
115 
116  // Ensure an old treefile is not still around:
117  std::remove((fileName_ + ".stc").c_str());
118  // Default treeCache size: 250
119  treeCache.CacheResize(250);
120 
121  return err;
122 }
123 
125  ASSERT(inUse);
126 
127  for (size_t i = 0, n = sortCaches_.size(); i < n; i++) {
128  delete sortCaches_[i].second;
129  }
130  sortCaches_.resize(0);
131 
132  errorT errGFile = codec_->flush();
133  errorT errIdx = idx->Close();
134  nb->Clear();
135  delete codec_;
136 
137  codec_ = NULL;
138  clear();
139  game->Clear();
140  fileMode_ = FMODE_None;
141  fileName_ = "<empty>";
142  gameNumber = -1;
143  gameAltered = false;
144  dbFilter->Init(0);
145  treeFilter->Init(0);
146  for (size_t i=0, n = filters_.size(); i < n; i++) delete filters_[i].second;
147  filters_.clear();
148  inUse = false;
149 
150  return (errIdx != OK) ? errIdx : errGFile;
151 }
152 
153 
154 void scidBaseT::clear() {
155  if (stats_ != NULL) { delete stats_; stats_ = NULL;}
156  if (duplicates_ != NULL) { delete[] duplicates_; duplicates_ = NULL; }
157  treeCache.Clear();
158  for (nameT nt = NAME_PLAYER; nt < NUM_NAME_TYPES; nt++) {
159  nameFreq_[nt].resize(0);
160  }
161 }
162 
164  for (size_t i = 0, n = sortCaches_.size(); i < n; ++i) {
165  sortCaches_[i].second->prepareForChanges();
166  }
167 }
168 
170  clear();
171  errorT res = codec_->flush();
172 
173  for (size_t i = 0, n = sortCaches_.size(); i < n; ++i) {
174  sortCaches_[i].second->checkForChanges(gNum);
175  }
176 
177  return res;
178 }
179 
180 errorT scidBaseT::getExtraInfo(const std::string& tagname, std::string* res) const {
181  if (tagname == "description") {
182  *res = idx->GetDescription();
183  } else if (tagname == "autoload") {
184  *res = to_string(idx->GetAutoLoad());
185  } else if (tagname == "type") {
186  *res = to_string(idx->GetType());
187  } else if (tagname.length() == 5 && tagname.find("flag") == 0) {
188  uint flagType = IndexEntry::CharToFlag(tagname[4]);
189  if (flagType == 0) return ERROR_BadArg;
190  const char* desc = idx->GetCustomFlagDesc(flagType);;
191  if (desc == 0) return ERROR_BadArg;
192  *res = desc;
193  } else {
194  return ERROR_BadArg;
195  }
196  return OK;
197 }
198 
199 errorT scidBaseT::setExtraInfo(const std::string& tagname, const char* new_value) {
200  if (tagname == "description") {
201  return idx->SetDescription(new_value);
202  } else if (tagname == "autoload") {
203  return idx->SetAutoLoad(strGetUnsigned(new_value));
204  } else if (tagname == "type") {
205  return idx->SetType(strGetUnsigned(new_value));
206  } else if (tagname.length() == 5 && tagname.find("flag") == 0) {
207  uint flagType = IndexEntry::CharToFlag(tagname[4]);
208  if (flagType == 0) return ERROR_BadArg;
209  if (idx->GetCustomFlagDesc(flagType) == 0) return ERROR_BadArg;
210  return idx->SetCustomFlagDesc(flagType, new_value);
211  }
212  return ERROR_BadArg;
213 }
214 
215 /**
216 * scidBaseT::makeGamePos() - constructs a GamePos object
217 * @game: a Game object with a valid current position
218 * @ravNum: current variation number
219 *
220 * This function extracts informations from the current position of Game @game
221 * and create a GamePos object with the corresponding informations:
222 * RAVdepth: current variation depth.
223 * RAVnum: current variation num.
224 * FEN: "Forsyth-Edwards Notation" describing the current position.
225 * NAGS: "Numeric Annotation Glyph" is a non-negative integer from 0 to 255
226 * used to indicate a simple annotation in a language independent manner.
227 * comment: text annotation of the current position.
228 * lastMoveSAN: the last move that was played to reach the current position.
229 * The move is indicated using English "Standard Algebraic Notation".
230 */
231 scidBaseT::GamePos scidBaseT::makeGamePos(Game& game, unsigned int ravNum) {
232  GamePos res;
233  res.RAVdepth = game.GetVarLevel();
234  res.RAVnum = ravNum;
235  char strBuf[256];
236  game.GetCurrentPos()->PrintFEN(strBuf, FEN_ALL_FIELDS);
237  res.FEN = strBuf;
238  for (byte* nag = game.GetNags(); *nag; nag++) {
239  res.NAGs.push_back(*nag);
240  }
241  res.comment = game.GetMoveComment();
242  game.GetPrevSAN(strBuf);
243  res.lastMoveSAN = strBuf;
244  return res;
245 }
246 
247 /**
248 * scidBaseT::getGame() - returns all the positions of a game
249 * @ie: a valid pointer to the IndexEntry of the desired game
250 * @dest: a container of GamePos objects where the positions will be stored.
251 *
252 * This function iterate all the positions of the game pointed by @ie and
253 * stores the positions in @dest. The container is NOT automatically cleared
254 * and the container should support push_back().
255 * The order of positions and of Recursive Annotation Variations (RAV)
256 * follows the pgn standard: "The alternate move sequence given by an RAV is
257 * one that may be legally played by first unplaying the move that appears
258 * immediately prior to the RAV. Because the RAV is a recursive construct,
259 * it may be nested."
260 * Each position have a RAVdepth and a RAVnum that allows to follow a
261 * variation from any given position:
262 * - skip all the next positions with a bigger RAVdepth
263 * - the variation ends with:
264 * - a lower RAVdepth or
265 * - an equal RAVdepth but different RAVnum or
266 * - the end of @dest
267 *
268 * Return OK if successful.
269 */
270 errorT scidBaseT::getGame(const IndexEntry* ie, std::vector<GamePos>& dest) {
271  ASSERT(ie != 0);
272 
273  // Create the Game object
274  ByteBuffer buf(BBUF_SIZE);
275  if (getGame(ie, &buf) != OK) {
276  return ERROR_Decode;
277  }
278  Game game;
279  if (game.Decode(&buf, GAME_DECODE_ALL) != OK) {
280  return ERROR_Decode;
281  }
282  std::vector<int> endPos = game.GetCurrentLocation();
283  game.MoveToPly(0);
284 
285  // Add start FEN
286  dest.push_back(makeGamePos(game, 0));
287 
288  // Iterate all the positions of the game
289  std::vector< std::pair<uint, uint> > rav;
290  rav.push_back(std::make_pair(0, 0));
291  errorT err = OK;
292  while (err == OK) {
293  uint nVariations = game.GetNumVariations();
294  err = game.MoveForward();
295  if (err == OK) {
296  dest.push_back(makeGamePos(game, rav.back().first));
297  }
298 
299  if (nVariations != 0) {
300  if (err == OK) {
301  // Go back in order to process variations
302  err = game.MoveBackup();
303  if (err != OK) break;
304  }
305  // Enter the first variation
306  err = game.MoveIntoVariation(0);
307  rav.push_back(std::make_pair(0, nVariations));
308  } else {
309  if (err == ERROR_EndOfMoveList) {
310  // Leave the current variation
311  err = game.MoveExitVariation();
312  if (err != OK) break;
313 
314  if (++rav.back().first < rav.back().second) {
315  // Enter the next variation
316  err = game.MoveIntoVariation(rav.back().first);
317  } else {
318  // All the sub-variation has been processed
319  rav.pop_back();
320  // Skip the main move of the parent variation
321  err = game.MoveForward();
322  }
323  }
324  }
325  }
326  if (rav.size() == 1 && game.GetCurrentLocation() == endPos) return OK;
327  return err;
328 }
329 
330 errorT scidBaseT::saveGame(Game* game, gamenumT replacedGameId) {
331  beginTransaction();
332  errorT err1 = saveGameHelper(game, replacedGameId);
333  errorT err2 = endTransaction(replacedGameId);
334  return (err1 != OK) ? err1 : err2;
335 }
336 
338  if (isReadOnly())
339  return ERROR_FileReadOnly;
340 
341  if (gameId < numGames())
342  return codec_->saveGame(game, gameId);
343 
344  errorT err = codec_->addGame(game);
345  if (err == OK)
346  extendFilters();
347  return err;
348 }
349 
351  if (srcBase == this) return ERROR_BadArg;
352  if (isReadOnly()) return ERROR_FileReadOnly;
353  if (gNum >= srcBase->numGames()) return ERROR_BadArg;
354 
355  beginTransaction();
356  errorT err = importGameHelper(srcBase, gNum);
357  errorT errClear = endTransaction();
358  return (err == OK) ? errClear : err;
359 }
360 
361 errorT scidBaseT::importGames(const scidBaseT* srcBase, const HFilter& filter, const Progress& progress) {
362  ASSERT(srcBase != 0);
363  ASSERT(filter != 0);
364  if (srcBase == this) return ERROR_BadArg;
365  if (isReadOnly()) return ERROR_FileReadOnly;
366 
367  beginTransaction();
368  errorT err = OK;
369  size_t iProgress = 0;
370  size_t totGames = filter->size();
371  for (gamenumT gNum = 0, n = srcBase->numGames(); gNum < n; gNum++) {
372  if (filter.get(gNum) == 0) continue;
373  err = importGameHelper(srcBase, gNum);
374  if (err != OK) break;
375  if (++iProgress % 8192 == 0) {
376  if (!progress.report(iProgress, totGames)) break;
377  }
378  }
379  errorT errClear = endTransaction();
380  return (err == OK) ? errClear : err;
381 }
382 
383 errorT scidBaseT::importGameHelper(const scidBaseT* sourceBase, uint gNum) {
384  const IndexEntry* srcIe = sourceBase->getIndexEntry(gNum);
385  uint gameDataLen = srcIe->GetLength();
386  const byte* gameData =
387  sourceBase->codec_->getGameData(srcIe->GetOffset(), gameDataLen);
388  if (gameData == 0) return ERROR_FileRead;
389 
390  IndexEntry ie = *srcIe;
391  errorT err;
392  err = ie.SetWhiteName(nb, srcIe->GetWhiteName(sourceBase->nb));
393  if (err != OK) return err;
394  err = ie.SetBlackName(nb, srcIe->GetBlackName(sourceBase->nb));
395  if (err != OK) return err;
396  err = ie.SetEventName(nb, srcIe->GetEventName(sourceBase->nb));
397  if (err != OK) return err;
398  err = ie.SetSiteName(nb, srcIe->GetSiteName(sourceBase->nb));
399  if (err != OK) return err;
400  err = ie.SetRoundName(nb, srcIe->GetRoundName(sourceBase->nb));
401  if (err != OK) return err;
402 
403  nb->AddElo(ie.GetWhite(), ie.GetWhiteElo());
404  nb->AddElo(ie.GetBlack(), ie.GetBlackElo());
405 
406  err = codec_->addGame(&ie, gameData, gameDataLen);
407  if (err == OK) extendFilters();
408  return err;
409 }
410 
411 /**
412  * Filters
413  */
414 std::string scidBaseT::newFilter() {
415  std::string newname = (filters_.size() == 0)
416  ? "a_"
417  : filters_.back().first;
418  if (newname[0] == 'z') {
419  newname = 'a' + newname;
420  } else {
421  newname = ++(newname[0]) + newname.substr(1);
422  }
423  filters_.push_back(std::make_pair(newname, new Filter(numGames())));
424  return newname;
425 }
426 
427 std::string scidBaseT::composeFilter(const std::string& mainFilter,
428  const std::string& maskFilter) const {
429  std::string res;
430  if (mainFilter.empty()) return res;
431 
432  if (mainFilter[0] != '+') {
433  res = mainFilter;
434  } else {
435  size_t maskName = mainFilter.find('+', 1);
436  if (maskName != std::string::npos)
437  res = mainFilter.substr(1, maskName - 1);
438  }
439 
440  if (!maskFilter.empty()) {
441  res = '+' + res + "+" + maskFilter;
442  }
443 
444  if (getFilter(res) == 0) res.clear();
445  return res;
446 }
447 
448 void scidBaseT::deleteFilter(const char* filterId) {
449  for (size_t i = 0, n = filters_.size(); i < n; i++) {
450  if (filters_[i].first == filterId) {
451  delete filters_[i].second;
452  filters_.erase(filters_.begin() + i);
453  break;
454  }
455  }
456 }
457 
458 void scidBaseT::extendFilters() {
459  dbFilter->Append(dbFilter->isWhole() ? 1 : 0);
460  treeFilter->Append(treeFilter->isWhole() ? 1 : 0);
461  for (size_t i = 0, n = filters_.size(); i < n; i++) {
462  Filter* filter = filters_[i].second;
463  filter->Append(filter->isWhole() ? 1 : 0);
464  }
465 }
466 
467 Filter* scidBaseT::fetchFilter(const std::string& filterId) const {
468  if (filterId == "dbfilter") return dbFilter;
469  if (filterId == "tree") return treeFilter;
470 
471  for (size_t i = 0, n = filters_.size(); i < n; i++) {
472  if (filterId == filters_[i].first)
473  return filters_[i].second;
474  }
475  return 0;
476 }
477 
478 HFilter scidBaseT::getFilterHelper(const std::string& filterId,
479  bool unmasked) const {
480  Filter* main = 0;
481  const Filter* mask = 0;
482  if (filterId.empty() || filterId[0] != '+') {
483  main = fetchFilter(filterId);
484  } else {
485  size_t maskName = filterId.find('+', 1);
486  if (maskName != std::string::npos) {
487  main = fetchFilter(filterId.substr(1, maskName - 1));
488  if (!unmasked) mask = fetchFilter(filterId.substr(maskName + 1));
489  }
490  }
491  return HFilter(main, mask);
492 }
493 
494 /**
495  * Statistics
496  */
498  if (stats_ == NULL) stats_ = new scidBaseT::Stats(this);
499  return *stats_;
500 }
501 
503 : count(0) {
504  std::fill_n(results, NUM_RESULT_TYPES, 0);
505 }
506 
509  minDate = ZERO_DATE;
510  maxDate = ZERO_DATE;
511  nYears = 0;
512  sumYears = 0;
513  std::fill_n(nResults, NUM_RESULT_TYPES, 0);
514  nRatings = 0;
515  sumRatings = 0;
516  minRating = 0;
517  maxRating = 0;
518 
519  // Read stats from index entry of each game:
520  for (gamenumT gnum=0, n = dbase->numGames(); gnum < n; gnum++) {
521  const IndexEntry* ie = dbase->getIndexEntry(gnum);
522  nResults[ie->GetResult()]++;
523  eloT elo = ie->GetWhiteElo();
524  if (elo > 0) {
525  nRatings++;
526  sumRatings += elo;
527  if (minRating == 0) { minRating = elo; }
528  if (elo < minRating) { minRating = elo; }
529  if (elo > maxRating) { maxRating = elo; }
530  }
531  elo = ie->GetBlackElo();
532  if (elo > 0) {
533  nRatings++;
534  sumRatings += elo;
535  if (minRating == 0) { minRating = elo; }
536  if (elo < minRating) { minRating = elo; }
537  if (elo > maxRating) { maxRating = elo; }
538  }
539  dateT date = ie->GetDate();
540  if (gnum == 0) {
541  maxDate = minDate = date;
542  }
543  if (date_GetYear(date) > 0) {
544  if (date < minDate) { minDate = date; }
545  if (date > maxDate) { maxDate = date; }
546  nYears++;
547  sumYears += date_GetYear (date);
548  }
549 
550  for (uint flag = 0; flag < IndexEntry::IDX_NUM_FLAGS; flag++) {
551  bool value = ie->GetFlag (1 << flag);
552  if (value) {
553  flagCount[flag]++;
554  }
555  }
556 
557  resultT result = ie->GetResult();
558  ecoT eco = ie->GetEcoCode();
559  if (eco == 0) {
560  ecoEmpty_.count++;
561  ecoEmpty_.results[result]++;
562  } else {
563  ecoValid_.count++;
564  ecoValid_.results[result]++;
565  eco = eco_Reduce(eco);
566  ecoStats_[eco].count++;
567  ecoStats_[eco].results[result]++;
568  eco /= 27;
569  ecoGroup3_[eco].count++;
570  ecoGroup3_[eco].results[result]++;
571  eco /= 10;
572  ecoGroup2_[eco].count++;
573  ecoGroup2_[eco].results[result]++;
574  eco /= 10;
575  ecoGroup1_[eco].count++;
576  ecoGroup1_[eco].results[result]++;
577  }
578  }
579 }
580 
581 const scidBaseT::Stats::Eco* scidBaseT::Stats::getEcoStats(const char* ecoStr) const {
582  ASSERT(ecoStr != 0);
583 
584  if (*ecoStr == 0) return &ecoValid_;
585 
586  ecoT eco = eco_FromString(ecoStr);
587  if (eco == 0) return 0;
588  eco = eco_Reduce(eco);
589 
590  switch(strlen(ecoStr)) {
591  case 0:
592  return &ecoValid_;
593  case 1:
594  return &(ecoGroup1_[eco / 2700]);
595  case 2:
596  return &(ecoGroup2_[eco / 270]);
597  case 3:
598  return &(ecoGroup3_[eco / 27]);
599  case 4:
600  case 5:
601  return &(ecoStats_[eco]);
602  }
603 
604  return 0;
605 }
606 
607 
608 
609 double scidBaseT::TreeStat::expVect_[1600];
610 
612 : toMove(NOCOLOR), resultW(0), resultD(0), resultB(0), exp(0), ngames(0), nexp(0)
613 {
614  if (TreeStat::expVect_[0] == 0) {
615  for (int i=-800; i < 800; i++) TreeStat::expVect_[i+800] = 1/(1 + pow(10, i/400.0));
616  }
617 }
618 
619 std::vector<scidBaseT::TreeStat> scidBaseT::getTreeStat(const HFilter& filter) {
620  ASSERT(filter != 0);
621 
622  std::vector<scidBaseT::TreeStat> res;
623  std::vector<FullMove> v;
624  for (gamenumT gnum = 0, n = numGames(); gnum < n; gnum++) {
625  uint ply = filter.get(gnum);
626  if (ply == 0) continue;
627  else ply--;
628 
629  const IndexEntry* ie = getIndexEntry(gnum);
631  if (move.isNull()) {
632  move = getGame(ie).getMove(ply);
633  }
634 
635  size_t i = 0;
636  while (i < v.size() && v[i] != move) i++;
637  if (i == v.size()) {
638  v.push_back(move);
639  res.push_back(scidBaseT::TreeStat());
640  }
641  res[i].add(ie->GetResult(), ie->GetWhiteElo(nb), ie->GetBlackElo(nb));
642  }
643 
644  for (size_t i = 0, n = v.size(); i < n; i++) {
645  res[i].SAN = (v[i].isNull()) ? "[end]" : v[i].getSAN(&(res[i].toMove));
646  }
647 
648  std::sort(res.begin(), res.end());
649  return res;
650 }
651 
653  uint* n_unused,
654  uint* n_sparse,
655  uint* n_badNameId) {
656  std::vector<uint> nbFreq[NUM_NAME_TYPES];
657  for (nameT n = NAME_PLAYER; n < NUM_NAME_TYPES; n++) {
658  nbFreq[n].resize(nb->GetNumNames(n), 0);
659  }
660 
661  uint last_offset = 0;
662  *n_sparse = 0;
663  *n_deleted = 0;
664  for (gamenumT i=0, n = numGames(); i < n; i++) {
665  const IndexEntry* ie = getIndexEntry (i);
666  if (ie->GetDeleteFlag()) { *n_deleted += 1; continue; }
667 
668  uint offset = ie->GetOffset();
669  if (offset < last_offset) *n_sparse += 1;
670  last_offset = offset;
671 
672  nbFreq[NAME_PLAYER][ie->GetWhite()] += 1;
673  nbFreq[NAME_PLAYER][ie->GetBlack()] += 1;
674  nbFreq[NAME_EVENT][ie->GetEvent()] += 1;
675  nbFreq[NAME_SITE][ie->GetSite()] += 1;
676  nbFreq[NAME_ROUND][ie->GetRound()] += 1;
677  }
678 
679  *n_unused = 0;
680  for (nameT n = NAME_PLAYER; n < NUM_NAME_TYPES; n++) {
681  *n_unused += std::count(nbFreq[n].begin(), nbFreq[n].end(), 0);
682  }
683 
684  *n_badNameId = idx->GetBadNameIdCount();
685  return OK;
686 }
687 
689  std::vector<std::string> filenames = codec_->getFilenames();
690  if (filenames.empty()) return ERROR_CodecUnsupFeat;
691 
692  if (fileMode_ != FMODE_Both) {
693  //Older scid version to be upgraded are opened read only
694  if (idx->GetVersion() == SCID_VERSION) return ERROR_FileMode;
695  }
696 
697  //1) Create a new temporary database
698  std::string filename = fileName_;
699  std::string tmpfile = filename + "__COMPACT__";
700  ICodecDatabase::Codec dbtype = codec_->getType();
701  scidBaseT tmp;
702  errorT err_Create = tmp.Open(dbtype, FMODE_Create, tmpfile.c_str());
703  if (err_Create != OK) return err_Create;
704 
705  //2) Copy the Index Header
706  tmp.beginTransaction();
707  tmp.idx->copyHeaderInfo(*idx);
708  gamenumT autoloadOld = idx->GetAutoLoad();
709  gamenumT autoloadNew = 1;
710 
711  //3) Create the list of games to be copied
712  typedef std::vector< std::pair<byte, uint> > sort_t;
713  sort_t sort;
714  uint n_deleted = 0;
715  for (gamenumT i = 0, n = numGames(); i < n; i++) {
716  const IndexEntry* ie = getIndexEntry(i);
717  if (ie->GetDeleteFlag()) {
718  n_deleted++;
719  continue;
720  }
721  byte stLine = ie->GetStoredLineCode();
722  sort.push_back(std::make_pair(stLine, i));
723  }
724  std::stable_sort(sort.begin(), sort.end());
725 
726  //4) Copy the games
727  uint iProgress = 0;
728  bool err_UserCancel = false;
729  errorT err_AddGame = OK;
730  for (sort_t::iterator it = sort.begin(); it != sort.end(); it++) {
731  err_AddGame = tmp.importGameHelper(this, (*it).second);
732  if (err_AddGame != OK) break;
733 
734  gamenumT oldGnum = it->second + 1;
735  if (oldGnum == autoloadOld) autoloadNew = tmp.numGames();
736  //TODO:
737  //- update bookmarks game number
738  // (*it).second == old game number
739  // tmp.numGames() == new game number
740  if (++iProgress % 8192 == 0) {
741  if (!progress.report(iProgress, sort.size())) {
742  err_UserCancel = true;
743  break;
744  }
745  }
746  }
747 
748  //5) Finalize the new database
749  tmp.idx->SetAutoLoad(autoloadNew);
750  std::vector<std::string> tmp_filenames = tmp.codec_->getFilenames();
751  errorT err_NbWrite = tmp.endTransaction();
752  errorT err_Close = tmp.Close();
753  if (err_Close == OK) err_Close = (filenames.size() == tmp_filenames.size()) ? OK : ERROR;
754 
755  //6) Error: cleanup and report
756  if (err_NbWrite != OK || err_Close != OK || err_UserCancel || err_AddGame != OK) {
757  for (size_t i = 0, n = tmp_filenames.size(); i < n; i++) {
758  std::remove(tmp_filenames[i].c_str());
759  }
760  if (err_AddGame != OK)
761  return err_AddGame;
762  if (err_UserCancel)
763  return ERROR_UserCancel;
764  if (err_NbWrite != OK)
765  return err_NbWrite;
766  ASSERT(err_Close != OK);
767  return err_Close;
768  }
769 
770  //7) Remember the active filters and SortCaches
771  std::vector<std::string> filters(filters_.size());
772  for (size_t i = 0, n = filters_.size(); i < n; i++) {
773  filters[i] = filters_[i].first;
774  }
775  std::vector< std::pair<std::string, int> > oldSC;
776  for (size_t i = 0, n = sortCaches_.size(); i < n; i++) {
777  int refCount = sortCaches_[i].second->incrRef(0);
778  if (refCount >= 0)
779  oldSC.push_back(std::make_pair(sortCaches_[i].first, refCount));
780  }
781 
782  //8) Remove the old database
783  if (Close() != OK) return ERROR_FileInUse;
784  for (size_t i = 0, n = filenames.size(); i < n; i++) {
785  if (std::remove(filenames[i].c_str()) != 0) return ERROR_CompactRemove;
786  }
787 
788  //9) Success: rename the files and open the new database
789  for (size_t i = 0, n = filenames.size(); i < n; i++) {
790  const char* s1 = tmp_filenames[i].c_str();
791  const char* s2 = filenames[i].c_str();
792  std::rename(s1, s2);
793  }
794  errorT res = Open(dbtype, FMODE_Both, filename.c_str());
795 
796  //10) Re-create filters and SortCaches
797  if (res == OK || res == ERROR_NameDataLoss) {
798  for (size_t i = 0, n = filters.size(); i < n; i++) {
799  filters_.push_back(
800  std::make_pair(filters[i], new Filter(numGames())));
801  }
802  for (size_t i = 0, n = oldSC.size(); i < n; i++) {
803  const std::string& criteria = oldSC[i].first;
804  SortCache* sc = SortCache::create(idx, nb, criteria.c_str());
805  if (sc != NULL) {
806  sc->incrRef(oldSC[i].second);
807  sortCaches_.push_back(std::make_pair(criteria, sc));
808  }
809  }
810  }
811 
812  return res;
813 }
814 
815 /**
816  * Retrieve a SortCache object matching the supplied @e criteria.
817  * A new SortCache with refCount equal to 0 is created if a suitable object is
818  * not found in @e sortCaches_. Objects with refCount <= 0 are destroyed by the
819  * @e releaseSortCache function independently from the provided @e criteria
820  * argument (implementing a rudimentary garbage collector).
821  * @param criteria: the list of fields by which games will be ordered.
822  * Each field should be followed by '+' to indicate an
823  * ascending order or by '-' for a descending order.
824  * @returns a pointer to a SortCache object in case of success, NULL otherwise.
825  */
826 SortCache* scidBaseT::getSortCache(const char* criteria) {
827  ASSERT(criteria != NULL);
828 
829  for (size_t i = 0, n = sortCaches_.size(); i < n; ++i) {
830  if (std::strcmp(criteria, sortCaches_[i].first.c_str()) == 0)
831  return sortCaches_[i].second;
832  }
833 
834  SortCache* sc = SortCache::create(idx, nb, criteria);
835  if (sc != NULL)
836  sortCaches_.push_back(std::pair<std::string, SortCache*>(criteria, sc));
837 
838  return sc;
839 }
840 
841 void scidBaseT::releaseSortCache(const char* criteria) {
842  size_t i = 0;
843  while (i < sortCaches_.size()) {
844  const char* tmp = sortCaches_[i].first.c_str();
845  int decr = std::strcmp(criteria, tmp) ? 0 : -1;
846  if (sortCaches_[i].second->incrRef(decr) <= 0) {
847  delete sortCaches_[i].second;
848  sortCaches_.erase(sortCaches_.begin() + i);
849  continue; //do not increment i
850  }
851  i += 1;
852  }
853 }
854 
855 SortCache* scidBaseT::createSortCache(const char* criteria) {
856  SortCache* sc = getSortCache(criteria);
857  if (sc != NULL)
858  sc->incrRef(1);
859 
860  return sc;
861 }
862 
863 size_t scidBaseT::listGames(const char* criteria, size_t start, size_t count,
864  const HFilter& filter, gamenumT* destCont) {
865  const SortCache* sc = getSortCache(criteria);
866  if (sc == NULL)
867  return 0;
868 
869  return sc->select(start, count, filter, destCont);
870 }
871 
872 size_t scidBaseT::sortedPosition(const char* criteria, const HFilter& filter,
873  gamenumT gameId) {
874  ASSERT(filter != NULL && filter->size() <= numGames());
875 
876  if (gameId >= numGames() || filter->get(gameId) == 0)
877  return INVALID_GAMEID;
878 
879  SortCache* sc = getSortCache(criteria);
880  if (sc == NULL)
881  return INVALID_GAMEID;
882 
883  return sc->sortedPosition(gameId, filter);
884 }
unsigned char byte
Definition: common.h:97
errorT SetAutoLoad(gamenumT gnum)
Definition: index.h:187
const errorT ERROR_CodecUnsupFeat
Definition: error.h:82
uint date_GetYear(dateT date)
Definition: date.h:55
Manages memory databases that do not have associated files.
Definition: codec_memory.h:47
byte resultT
Definition: common.h:183
void deleteFilter(const char *filterId)
Definition: scidbase.cpp:448
errorT MoveIntoVariation(uint varNumber)
Definition: game.cpp:863
bool report(size_t done, size_t total) const
Definition: misc.h:136
bool GetFlag(uint32_t mask) const
Definition: indexentry.h:304
const errorT ERROR_EndOfMoveList
Definition: error.h:66
void PrintFEN(char *str, uint flags)
Definition: position.cpp:2975
idNumberT GetEvent() const
Definition: indexentry.h:127
FullMove getMove(int ply_to_skip)
Definition: fastgame.h:273
uint32_t GetLength() const
Definition: indexentry.h:89
Definition: indexentry.h:400
byte * GetNags()
Definition: game.h:391
bool isWhole() const
Definition: filter.h:54
const errorT OK
Definition: error.h:23
const uint NUM_RESULT_TYPES
Definition: common.h:182
size_t select(size_t row_offset, size_t row_count, const HFilter &filter, gamenumT *result) const
Retrieve the sorted list of games&#39; ids.
Definition: sortcache.cpp:160
unsigned int RAVdepth
Definition: scidbase.h:128
const errorT ERROR_FileInUse
Definition: error.h:37
const colorT NOCOLOR
Definition: common.h:205
SortCache * createSortCache(const char *criteria)
Increment the reference count of a SortCache object matching criteria.
Definition: scidbase.cpp:855
uint dateT
Definition: common.h:155
static FullMove getMove(uint code, uint ply=0)
Definition: stored.h:38
#define ASSERT(f)
Definition: common.h:67
const errorT ERROR_BadArg
Definition: error.h:28
uint GetVarLevel()
Definition: game.h:415
gamenumT numGames() const
Definition: scidbase.h:97
int incrRef(int incr)
Definition: sortcache.h:150
const IndexEntry * getIndexEntry(gamenumT g) const
Definition: scidbase.h:101
Definition: misc.h:124
NameBase * nb
Definition: scidbase.h:281
std::string to_string(int val)
Definition: misc.h:153
errorT Close()
Definition: scidbase.cpp:124
virtual std::vector< std::string > getFilenames()=0
Returns the full path of the files used by the database.
uint64_t sumYears
Definition: scidbase.h:43
unsigned int RAVnum
Definition: scidbase.h:129
errorT saveGame(Game *game, gamenumT replacedGameId=INVALID_GAMEID)
Add or replace a game into the database.
Definition: scidbase.cpp:330
eloT GetBlackElo() const
Definition: indexentry.h:252
errorT endTransaction(gamenumT gameId=INVALID_GAMEID)
Update caches and flush the database&#39;s files.
Definition: scidbase.cpp:169
const gamenumT INVALID_GAMEID
Definition: scidbase.h:35
Index * idx
Definition: scidbase.h:280
static uint CharToFlag(char ch)
Definition: indexentry.h:670
resultT GetResult() const
Definition: indexentry.h:245
uint results[NUM_RESULT_TYPES]
Definition: scidbase.h:54
errorT SetSiteName(NameBase *nb, const char *s)
Definition: indexentry.h:217
#define BBUF_SIZE
Definition: common.h:42
void MoveToPly(ushort hmNumber)
Definition: game.cpp:830
std::string comment
Definition: scidbase.h:132
errorT importGame(const scidBaseT *srcBase, uint gNum)
Definition: scidbase.cpp:350
const dateT ZERO_DATE
Definition: date.h:34
ecoT GetEcoCode() const
Definition: indexentry.h:264
void beginTransaction()
This function must be called before modifying the games of the database.
Definition: scidbase.cpp:163
const errorT ERROR_CompactRemove
Definition: error.h:42
Defines the SortCache class, which sorts the games of an Index.
errorT setExtraInfo(const std::string &tagname, const char *new_value)
Definition: scidbase.cpp:199
std::string newFilter()
Filters : unique identifier of a Filter.
Definition: scidbase.cpp:414
const Stats & getStats() const
Statistics.
Definition: scidbase.cpp:497
uint64_t sumRatings
Definition: scidbase.h:46
int main(int argc, char *argv[])
Definition: tkscid.cpp:81
const errorT ERROR_FileRead
Definition: error.h:33
uint nameT
Definition: namebase.h:29
ecoT eco_Reduce(ecoT eco)
ecoReduce() - maps eco to a smaller set : the eco value to convert (must be != 0) ...
Definition: misc.cpp:163
errorT SetEventName(NameBase *nb, const char *s)
Definition: indexentry.h:211
const char * GetBlackName(const NameBase *nb) const
Definition: indexentry.h:186
bool GetDeleteFlag() const
Definition: indexentry.h:318
const errorT ERROR_UserCancel
Definition: error.h:27
static ICodecDatabase * make(Codec codec, errorT *err, fileModeT fMode, const char *filename, const Progress &progress, Index *idx, NameBase *nb)
Creates a new object and calls the virtual function dyn_open().
Definition: scidbase.cpp:29
sort?type?
Definition: analysis.tcl:321
~scidBaseT()
Definition: scidbase.cpp:75
This interface separates the logic of a database from its representation.
Definition: codec.h:43
size_t sortedPosition(const char *criteria, const HFilter &filter, gamenumT gameId)
Get the sorted position of a game.
Definition: scidbase.cpp:872
virtual const byte * getGameData(uint32_t offset, uint32_t length)=0
Fetches the data of a game (excluding index info), encoded in native format.
uint32_t uint
Definition: common.h:99
errorT saveGameHelper(Game *game, gamenumT gameId)
Definition: scidbase.cpp:337
Definition: move.tcl:20
Stats(const scidBaseT *dbase)
Definition: scidbase.cpp:507
const char * GetSiteName(const NameBase *nb) const
Definition: indexentry.h:192
Definition: index.h:61
uint32_t GetOffset() const
Definition: indexentry.h:87
errorT getCompactStat(uint *n_deleted, uint *n_unused, uint *n_sparse, uint *n_badNameId)
Definition: scidbase.cpp:652
std::vector< int > GetCurrentLocation()
Definition: game.cpp:4001
errorT getExtraInfo(const std::string &tagname, std::string *res) const
Definition: scidbase.cpp:180
ushort eloT
Definition: common.h:160
errorT MoveBackup()
Definition: game.cpp:813
errorT SetRoundName(NameBase *nb, const char *s)
Definition: indexentry.h:223
void releaseSortCache(const char *criteria)
Decrement the reference count of the SortCache object matching criteria.
Definition: scidbase.cpp:841
Position * GetCurrentPos()
Definition: game.h:348
void Append(byte value)
Definition: filter.h:157
ushort ecoT
Definition: common.h:161
std::vector< int > NAGs
Definition: scidbase.h:131
#define GAME_DECODE_ALL
Definition: game.h:185
errorT MoveForward()
Definition: game.cpp:796
uint32_t strGetUnsigned(const char *str)
Definition: misc.h:276
errorT copyHeaderInfo(const Index &src)
Header setter functions.
Definition: index.h:156
unsigned short errorT
Definition: error.h:20
size_t size() const
Definition: hfilter.h:161
const Eco * getEcoStats(const char *ecoStr) const
Definition: scidbase.cpp:581
uint nResults[NUM_RESULT_TYPES]
Definition: scidbase.h:44
Definition: errors.tcl:17
uint flagCount[IndexEntry::IDX_NUM_FLAGS]
Definition: scidbase.h:39
idNumberT GetNumNames(nameT n) const
Definition: namebase.h:107
errorT importGames(const scidBaseT *srcBase, const HFilter &filter, const Progress &progress)
Definition: scidbase.cpp:361
const errorT ERROR_FileReadOnly
Definition: error.h:41
errorT SetWhiteName(NameBase *nb, const char *s)
Definition: indexentry.h:199
errorT compact(const Progress &progress)
Definition: scidbase.cpp:688
This class sorts games contained into an Index.
Definition: sortcache.h:50
errorT MoveExitVariation()
Definition: game.cpp:888
This class manages databases encoded in Scid format v4.
Definition: codec_scid4.h:39
clear?pattern?
We don&#39;t remove TooltipMenu because there.
int GetBadNameIdCount() const
GetBadNameIdCount() - return the number of invalid name handles.
Definition: index.h:112
Definition: game.h:227
idNumberT GetRound() const
Definition: indexentry.h:141
std::vector< scidBaseT::TreeStat > getTreeStat(const HFilter &filter)
Definition: scidbase.cpp:619
versionT GetVersion() const
Definition: index.h:143
gamenumT GetAutoLoad() const
Definition: index.h:149
const errorT ERROR_Decode
Definition: error.h:71
errorT SetBlackName(NameBase *nb, const char *s)
Definition: indexentry.h:205
std::string lastMoveSAN
Definition: scidbase.h:133
size_t sortedPosition(gamenumT gameId, const HFilter &filter) const
Get the sorted position of a game.
Definition: sortcache.cpp:213
byte get(gamenumT gnum) const
Definition: hfilter.h:168
idNumberT GetBlack() const
Definition: indexentry.h:116
ecoT eco_FromString(const char *ecoStr)
Definition: misc.cpp:69
Definition: filter.h:32
const char * GetEventName(const NameBase *nb) const
Definition: indexentry.h:189
removewin args
Definition: board.tcl:968
const char * GetMoveComment()
Definition: game.h:374
uint GetNumVariations()
Definition: game.h:416
static SortCache * create(const Index *idx, const NameBase *nb, const char *criteria)
Create a new SortCache object, builds the hash table, and asynchronously sorts all the games...
Definition: sortcache.cpp:116
errorT Open(ICodecDatabase::Codec dbtype, fileModeT mode, const char *filename=0, const Progress &progress=Progress())
Definition: scidbase.cpp:89
uint gamenumT
Definition: common.h:159
const versionT SCID_VERSION
Definition: common.h:49
fileModeT
Definition: common.h:144
const errorT ERROR_FileMode
Definition: error.h:38
eloT GetWhiteElo() const
Definition: indexentry.h:246
const errorT ERROR
Definition: error.h:26
Definition: tree.tcl:6
void GetPrevSAN(char *str)
Definition: game.cpp:1977
const errorT ERROR_NameDataLoss
Definition: error.h:54
uint64_t nYears
Definition: scidbase.h:42
Implements the CodecScid4 class, which manages the databases encoded in Scid format version 4...
std::string FEN
Definition: scidbase.h:130
idNumberT GetSite() const
Definition: indexentry.h:134
std::string composeFilter(const std::string &mainFilter, const std::string &maskFilter) const
Definition: scidbase.cpp:427
idNumberT GetWhite() const
Definition: indexentry.h:109
const char * GetRoundName(const NameBase *nb) const
Definition: indexentry.h:195
bool isNull() const
Definition: fullmove.h:51
dateT GetDate() const
Definition: indexentry.h:231
byte GetStoredLineCode() const
Definition: indexentry.h:330
const char * GetWhiteName(const NameBase *nb) const
Definition: indexentry.h:183
Definition: indexentry.h:54
errorT Decode(ByteBuffer *buf, byte flags)
Definition: game.cpp:3931
FastGame getGame(const IndexEntry *ie) const
Definition: scidbase.h:111
Implements the CodecPgn class, which manages the databases encoded in PGN format. ...
Implements the CodecMemory class, which manages the memory representation of the open databases...
size_t listGames(const char *criteria, size_t start, size_t count, const HFilter &filter, gamenumT *destCont)
Retrieve a list of ordered game indexes sorted by criteria.
Definition: scidbase.cpp:863
const uint FEN_ALL_FIELDS
Definition: position.h:50