Line data Source code
1 : /*
2 : # Copyright (C) 2014-2016 Fulvio Benini
3 :
4 : * This file is part of Scid (Shane's Chess Information Database).
5 : *
6 : * Scid is free software: you can redistribute it and/or modify
7 : * it under the terms of the GNU General Public License as published by
8 : * the Free Software Foundation.
9 : *
10 : * Scid is distributed in the hope that it will be useful,
11 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : * GNU General Public License for more details.
14 : *
15 : * You should have received a copy of the GNU General Public License
16 : * along with Scid. If not, see <http://www.gnu.org/licenses/>.
17 : */
18 :
19 : #ifndef SCIDBASE_H
20 : #define SCIDBASE_H
21 :
22 : #include "bytebuf.h"
23 : #include "codec.h"
24 : #include "containers.h"
25 : #include "fastgame.h"
26 : #include "game.h"
27 : #include "index.h"
28 : #include "namebase.h"
29 : #include "tree.h"
30 : #include <array>
31 : #include <vector>
32 : #include <memory>
33 :
34 : class SortCache;
35 :
36 :
37 : const gamenumT INVALID_GAMEID = 0xffffffff;
38 :
39 : struct scidBaseT {
40 : struct Stats {
41 : uint flagCount[IndexEntry::IDX_NUM_FLAGS]; // Num of games with each flag set.
42 : dateT minDate;
43 : dateT maxDate;
44 : uint64_t nYears;
45 : uint64_t sumYears;
46 : uint nResults [NUM_RESULT_TYPES];
47 : uint nRatings;
48 : uint64_t sumRatings;
49 : uint minRating;
50 : uint maxRating;
51 :
52 : Stats(const scidBaseT* dbase);
53 :
54 : struct Eco {
55 : uint count;
56 : uint results [NUM_RESULT_TYPES];
57 :
58 : Eco();
59 : };
60 : const Eco* getEcoStats(const char* ecoStr) const;
61 :
62 : private:
63 : Eco ecoEmpty_;
64 : Eco ecoValid_;
65 : Eco ecoStats_ [(1 + (1<<16)/131)*27];
66 : Eco ecoGroup1_[(1 + (1<<16)/131)/100];
67 : Eco ecoGroup2_[(1 + (1<<16)/131)/10];
68 : Eco ecoGroup3_[(1 + (1<<16)/131)];
69 : };
70 :
71 0 : struct TreeStat {
72 : colorT toMove;
73 : std::string SAN;
74 : int resultW, resultD, resultB;
75 : double exp;
76 : int ngames, nexp;
77 :
78 : public:
79 : TreeStat();
80 : void add(int result, int eloW, int eloB);
81 0 : bool operator<(const TreeStat& cmp) const { return ngames > cmp.ngames; }
82 :
83 : private:
84 : static double expVect_[1600];
85 : };
86 :
87 : scidBaseT();
88 : ~scidBaseT();
89 :
90 : errorT Open(ICodecDatabase::Codec dbtype,
91 : fileModeT mode,
92 : const char* filename = 0,
93 : const Progress& progress = Progress());
94 :
95 : errorT Close ();
96 :
97 : const std::string& getFileName() const { return fileName_; }
98 12 : bool isReadOnly() const { return fileMode_ == FMODE_ReadOnly; }
99 6439 : gamenumT numGames() const { return idx->GetNumGames(); }
100 :
101 : /// Returns a vector of tag pairs containing extra information about the
102 : /// database (type, description, autoload, etc..)
103 : std::vector<std::pair<const char*, std::string> > getExtraInfo() const {
104 : return codec_->getExtraInfo();
105 : }
106 :
107 : /// Store an extra information about the database (type, description, etc..)
108 : errorT setExtraInfo(const char* tagname, const char* new_value) {
109 : // TODO: move the code to CodecMemory and CodecSCID4
110 : if (std::strcmp(tagname, "type") == 0)
111 : return idx->SetType(strGetUnsigned(new_value));
112 :
113 : if (codec_->getType() == ICodecDatabase::SCID4) {
114 : if (std::strcmp(tagname, "description") == 0)
115 : return idx->SetDescription(new_value);
116 :
117 : if (std::strcmp(tagname, "autoload") == 0)
118 : return idx->SetAutoLoad(strGetUnsigned(new_value));
119 :
120 : auto len = std::strlen(tagname);
121 : if (len == 5 && std::equal(tagname, tagname + 4, "flag")) {
122 : uint flagType = IndexEntry::CharToFlag(tagname[4]);
123 : if (flagType != 0 &&
124 : idx->GetCustomFlagDesc(flagType) != nullptr)
125 : return idx->SetCustomFlagDesc(flagType, new_value);
126 : }
127 : }
128 : return ERROR_CodecUnsupFeat;
129 : }
130 :
131 1390 : const IndexEntry* getIndexEntry(gamenumT g) const {
132 1390 : return idx->GetEntry(g);
133 : }
134 34 : const IndexEntry* getIndexEntry_bounds(gamenumT g) const {
135 34 : if (g < numGames()) return getIndexEntry(g);
136 4 : return 0;
137 : }
138 15137 : const NameBase* getNameBase() const {
139 15137 : return nb_;
140 : }
141 0 : FastGame getGame(const IndexEntry* ie) const {
142 0 : uint length = ie->GetLength();
143 0 : const byte* b = codec_->getGameData(ie->GetOffset(), length);
144 0 : if (b == 0) length = 0; // Error
145 0 : return FastGame::Create(b, b + length);
146 : }
147 18 : errorT getGame(const IndexEntry* ie, ByteBuffer* destBuf) const {
148 18 : uint length = ie->GetLength();
149 18 : const byte* b = codec_->getGameData(ie->GetOffset(), length);
150 18 : if (b == 0) return ERROR_FileRead;
151 : // The data for the game is not actually copied into the bytebuffer, which would
152 : // be slower and a waste of time if the bytebuffer is not going to be modified.
153 18 : destBuf->ProvideExternal(const_cast<byte*>(b), length);
154 18 : return OK;
155 : }
156 12 : errorT getGame(const IndexEntry& ie, Game& dest) const {
157 24 : ByteBuffer buf(0);
158 12 : auto res = getGame(&ie, &buf);
159 24 : return (res != OK) ? res : dest.Decode(&buf, GAME_DECODE_ALL);
160 : }
161 :
162 : errorT importGame(const scidBaseT* srcBase, uint gNum);
163 : errorT importGames(const scidBaseT* srcBase, const HFilter& filter,
164 : const Progress& progress);
165 : errorT importGames(ICodecDatabase::Codec dbtype, const char* filename,
166 : const Progress& progress, std::string& errorMsg);
167 :
168 : /**
169 : * Add or replace a game into the database.
170 : * @param game: valid pointer to a Game object with the data of the game.
171 : * @param replacedGameId: id of the game to replace.
172 : * If >= numGames(), a new game will be added.
173 : * @returns OK if successful or an error code.
174 : */
175 : errorT saveGame(Game* game, gamenumT replacedGameId = INVALID_GAMEID);
176 : // TODO: private:
177 : errorT saveGameHelper(Game* game, gamenumT gameId);
178 :
179 :
180 : bool getFlag(uint flag, uint gNum) const {
181 : return idx->GetEntry(gNum)->GetFlag (flag);
182 : }
183 : errorT setFlag(bool value, uint flag, uint gNum);
184 : errorT setFlag(bool value, uint flag, const HFilter& filter);
185 : errorT invertFlag(uint flag, uint gNum);
186 : errorT invertFlag(uint flag, const HFilter& filter);
187 :
188 : /**
189 : * A Filter is a selection of games, usually obtained searching the
190 : * database. A new Filter is created calling the function newFilter()
191 : * and must be released calling the function deleteFilter().
192 : * A composed Filter is a special construct created combining two Filters
193 : * and includes only the games contained in both Filters.
194 : * A composed Filter should NOT be released.
195 : */
196 : std::string newFilter();
197 : std::string composeFilter(const std::string& mainFilter,
198 : const std::string& maskFilter) const;
199 : void deleteFilter(const char* filterId);
200 36551 : HFilter getFilter(const std::string& filterId) const {
201 36551 : return getFilterHelper(filterId, false);
202 : }
203 : HFilter getMainFilter(const std::string& filterId) const {
204 : return getFilterHelper(filterId, true);
205 : }
206 :
207 : const Stats& getStats() const;
208 : std::vector<scidBaseT::TreeStat> getTreeStat(const HFilter& filter);
209 : uint getNameFreq (nameT nt, idNumberT id) {
210 : if (nameFreq_[nt].size() == 0)
211 : nameFreq_ = idx->calcNameFreq(*getNameBase());
212 : return nameFreq_[nt][id];
213 : }
214 :
215 : errorT getCompactStat(unsigned long long* n_deleted,
216 : unsigned long long* n_unused,
217 : unsigned long long* n_sparse,
218 : unsigned long long* n_badNameId);
219 : errorT compact(const Progress& progress);
220 :
221 :
222 : /**
223 : * Increment the reference count of a SortCache object matching @e criteria.
224 : * @param criteria: the list of fields by which games will be ordered.
225 : * Each field should be followed by '+' to indicate an
226 : * ascending order or by '-' for a descending order.
227 : * @returns a pointer to a SortCache object in case of success, NULL
228 : * otherwise.
229 : */
230 : SortCache* createSortCache(const char* criteria);
231 :
232 : /**
233 : * Decrement the reference count of the SortCache object matching @e
234 : * criteria. Cached objects with refCount <= 0 are destroyed independently
235 : * from the value of @e criteria.
236 : * @param criteria: the list of fields by which games will be ordered.
237 : * Each field should be followed by '+' to indicate an
238 : * ascending order or by '-' for a descending order.
239 : */
240 : void releaseSortCache(const char* criteria);
241 :
242 : /**
243 : * Retrieve a list of ordered game indexes sorted by @e criteria.
244 : * This function will be much faster if a SortCache object matching @e
245 : * criteria already exists (previously created with @e createSortCache).
246 : * @param criteria: the list of fields by which games will be ordered.
247 : * Each field should be followed by '+' to indicate an
248 : * ascending order or by '-' for a descending order.
249 : * @param start: the offset of the first row to return.
250 : * The offset of the initial row is 0.
251 : * @param count: maximum number of rows to return.
252 : * @param filter: a reference to a valid (!= NULL) HFilter object.
253 : * Games not included into the filter will be ignored.
254 : * @param[out] destCont: valid pointer to an array where the sorted list of
255 : * games will be stored (should be able to contain at
256 : * least @e count elements).
257 : * @returns the number of games' ids stored into @e destCont.
258 : */
259 : size_t listGames(const char* criteria, size_t start, size_t count,
260 : const HFilter& filter, gamenumT* destCont);
261 :
262 : /**
263 : * Get the sorted position of a game.
264 : * This function will be much faster if a SortCache object matching @e
265 : * criteria already exists (previously created with @e createSortCache).
266 : * @param criteria: the list of fields by which games will be ordered.
267 : * Each field should be followed by '+' to indicate an
268 : * ascending order or by '-' for a descending order.
269 : * @param filter: a reference to a valid (!= NULL) HFilter object.
270 : * Games not included into the filter will be ignored.
271 : * @param gameId: the id of the game.
272 : * @returns the sorted position of @e gameId.
273 : */
274 : size_t sortedPosition(const char* criteria, const HFilter& filter,
275 : gamenumT gameId);
276 :
277 : void setDuplicates(uint* duplicates) {
278 : if (duplicates_ != NULL) { delete[] duplicates_; duplicates_ = NULL; }
279 : duplicates_ = duplicates;
280 : }
281 : uint getDuplicates(gamenumT gNum) {
282 : return (duplicates_ == NULL) ? 0 : duplicates_[gNum];
283 : }
284 :
285 : /**
286 : * Transform the IndexEntries of the games included in @e hfilter.
287 : * The @e entry_op must accept a IndexEntry& parameter and return true when
288 : * the IndexEntry was modified.
289 : * @param hfilter: HFilter containing the games to be transformed.
290 : * @param progress: a Progress object used for GUI communications.
291 : * @param entry_op: operator that will be applied to games' IndexEntry.
292 : * @returns a std::pair containing OK (or an error code) and the number of
293 : * games modified.
294 : */
295 : template <typename TOper>
296 : std::pair<errorT, size_t>
297 : transformIndex(HFilter hfilter, const Progress& progress, TOper entry_op) {
298 : beginTransaction();
299 : auto res = transformIndex_(hfilter, progress, entry_op);
300 : auto err = endTransaction();
301 : res.first = (res.first == OK) ? err : res.first;
302 : return res;
303 : }
304 :
305 : /**
306 : * Transform the names of the games included in @e hfilter.
307 : * The function @e getID maps all the old idNumberT to the new idNumberT.
308 : * It's invoked for each game and must accept as parameters a idNumberT and
309 : * a const IndexEntry&; must return the (eventually different) idNumberT.
310 : * @param nt: type of the names to be modified.
311 : * @param hfilter: HFilter containing the games to be transformed.
312 : * @param progress: a Progress object used for GUI communications.
313 : * @param newNames: optional vector of names to be added to the database.
314 : * @param fnInit: function that is invoked before beginning the
315 : * transformation; must accept a vector that contains the
316 : * idNumberTs of the names in @e newNames.
317 : * @param getID: function that maps the old idNumberTs to the new ones.
318 : * @returns a std::pair containing OK (or an error code) and the number of
319 : * games modified.
320 : */
321 : template <typename TInitFunc, typename TMapFunc>
322 : std::pair<errorT, size_t>
323 : transformNames(nameT nt, HFilter hfilter, const Progress& progress,
324 : const std::vector<std::string>& newNames, TInitFunc fnInit,
325 : TMapFunc getID);
326 :
327 : // TODO: private:
328 : /**
329 : * This function must be called before modifying the games of the database.
330 : * Currently this function do not guarantees that the database is not altered
331 : * in case of errors.
332 : */
333 : void beginTransaction();
334 :
335 : // TODO: private:
336 : /**
337 : * Update caches and flush the database's files.
338 : * This function must be called after changing one or more games.
339 : * @param gameId: id of the modified game
340 : * INVALID_GAMEID to update all games.
341 : * @returns OK if successful or an error code.
342 : */
343 : errorT endTransaction(gamenumT gameId = INVALID_GAMEID);
344 :
345 : public:
346 : Index* idx; // the Index file in memory for this base.
347 : bool inUse; // true if the database is open (in use).
348 : treeT tree;
349 : TreeCache treeCache;
350 : ByteBuffer* bbuf;
351 : Filter* dbFilter;
352 : Filter* treeFilter;
353 :
354 :
355 : //TODO: this vars do not belong to scidBaseT class
356 : Game* game; // the active game for this base.
357 : int gameNumber; // game number of active game.
358 : bool gameAltered; // true if game is modified
359 : UndoRedo<Game, 100> gameAlterations;
360 : std::pair<Game*, bool> deprecated_push_pop;
361 :
362 : private:
363 : std::unique_ptr<ICodecDatabase> codec_;
364 : NameBase* nb_;
365 : std::string fileName_; // File name without ".si" suffix
366 : fileModeT fileMode_; // Read-only, write-only, or both.
367 : std::vector< std::pair<std::string, Filter*> > filters_;
368 : mutable Stats* stats_;
369 : std::array<std::vector<int>, NUM_NAME_TYPES> nameFreq_;
370 : uint* duplicates_; // For each game: idx of duplicate game + 1 (0 if there is no duplicate).
371 : std::vector< std::pair<std::string, SortCache*> > sortCaches_;
372 :
373 : private:
374 : void clear();
375 : errorT importGameHelper(const scidBaseT* sourceBase, uint gNum);
376 :
377 : Filter* fetchFilter(const std::string& filterId) const;
378 : HFilter getFilterHelper(const std::string& filterId,
379 : bool unmasked = false) const;
380 : SortCache* getSortCache(const char* criteria);
381 :
382 : /**
383 : * Apply a transform operator to games' IndexEntry included in @e hfilter.
384 : * The @entry_op should accept a IndexEntry& parameter and return true when
385 : * the IndexEntry was modified.
386 : * @param hfilter: HFilter containing the games to be transformed.
387 : * @param progress: a Progress object used for GUI communications.
388 : * @param entry_op: operator that will be applied to games' IndexEntry.
389 : * @returns a std::pair containing OK (or an error code) and the number of
390 : * games modified.
391 : */
392 : template <typename TOper>
393 : std::pair<errorT, size_t>
394 : transformIndex_(HFilter hfilter, const Progress& progress, TOper entry_op) {
395 : size_t nCorrections = 0;
396 : size_t iProg = 0;
397 : size_t totProg = hfilter->size();
398 : for (auto& gnum : hfilter) {
399 : if ((++iProg % 8192 == 0) && !progress.report(iProg, totProg))
400 : return std::make_pair(ERROR_UserCancel, nCorrections);
401 :
402 : IndexEntry newIE = *getIndexEntry(gnum);
403 : if (!entry_op(newIE))
404 : continue;
405 :
406 : auto err = codec_->saveIndexEntry(newIE, gnum);
407 : if (err != OK)
408 : return std::make_pair(err, nCorrections);
409 :
410 : ++nCorrections;
411 : }
412 : return std::make_pair(OK, nCorrections);
413 : }
414 : };
415 :
416 0 : inline void scidBaseT::TreeStat::add(int result, int eloW, int eloB) {
417 0 : ngames++;
418 0 : double r = 0;
419 0 : switch (result) {
420 0 : case RESULT_White: resultW++; r = 1; break;
421 0 : case RESULT_Draw: resultD++; r = 0.5; break;
422 0 : case RESULT_Black: resultB++; break;
423 0 : default: return;
424 : }
425 0 : if (eloW == 0 || eloB == 0) return;
426 0 : int eloDiff = eloB - eloW;
427 0 : if (eloDiff < 800 && eloDiff >= -800) {
428 0 : exp += r - expVect_[eloDiff+800];
429 0 : nexp++;
430 : }
431 : }
432 :
433 : inline errorT scidBaseT::invertFlag(uint flag, uint gNum) {
434 : return setFlag(! getFlag(flag, gNum), flag, gNum);
435 : }
436 :
437 : inline errorT scidBaseT::invertFlag(uint flag, const HFilter& filter) {
438 : errorT res = OK;
439 : for (gamenumT i = 0, n = numGames(); i < n; i++) {
440 : if (filter != 0 && filter->get(i) == 0) continue;
441 : res = invertFlag(flag, i);
442 : if (res != OK) return res;
443 : }
444 : return res;
445 : }
446 :
447 : inline errorT scidBaseT::setFlag(bool value, uint flag, uint gNum){
448 : ASSERT(gNum < idx->GetNumGames());
449 : IndexEntry* ie = idx->FetchEntry (gNum);
450 : ie->SetFlag (flag, value);
451 : errorT res = idx->WriteEntry(ie, gNum);
452 : if (stats_ != NULL) { delete stats_; stats_ = NULL;}
453 : // TODO: necessary only for sortcaches with SORTING_deleted (and SORTING_flags when implemented)
454 : // idx->IndexUpdated(gNum);
455 : return res;
456 : }
457 :
458 : inline errorT scidBaseT::setFlag(bool value, uint flag, const HFilter& filter) {
459 : errorT res = OK;
460 : for (gamenumT gNum = 0, n = numGames(); gNum < n; gNum++) {
461 : if (filter != 0 && filter->get(gNum) == 0) continue;
462 : res = setFlag(value, flag, gNum);
463 : if (res != OK) return res;
464 : }
465 : return res;
466 : }
467 :
468 :
469 : template <typename TInitFunc, typename TMapFunc>
470 : std::pair<errorT, size_t>
471 : scidBaseT::transformNames(nameT nt, HFilter hfilter, const Progress& progress,
472 : const std::vector<std::string>& newNames,
473 : TInitFunc initFunc, TMapFunc getID) {
474 : beginTransaction();
475 :
476 : std::vector<idNumberT> nameIDs(newNames.size());
477 : auto it = nameIDs.begin();
478 : for (auto& name : newNames) {
479 : auto id = codec_->addName(nt, name.c_str());
480 : if (id.first != OK) {
481 : endTransaction();
482 : return std::make_pair(id.first, size_t(0));
483 : }
484 : *it++ = id.second;
485 : }
486 :
487 : initFunc(nameIDs);
488 :
489 : auto fnGet = [](nameT nt, const IndexEntry& ie) {
490 : switch (nt) { // clang-format off
491 : case NAME_PLAYER: return ie.GetWhite();
492 : case NAME_EVENT: return ie.GetEvent();
493 : case NAME_SITE: return ie.GetSite();
494 : } // clang-format on
495 : ASSERT(nt == NAME_ROUND);
496 : return ie.GetRound();
497 : };
498 : auto fnSet = [](nameT nt, IndexEntry& ie, idNumberT newID) {
499 : switch (nt) { // clang-format off
500 : case NAME_PLAYER: return ie.SetWhite(newID);
501 : case NAME_EVENT: return ie.SetEvent(newID);
502 : case NAME_SITE: return ie.SetSite(newID);
503 : } // clang-format on
504 : ASSERT(nt == NAME_ROUND);
505 : return ie.SetRound(newID);
506 : };
507 : auto res = transformIndex_(hfilter, progress, [&](IndexEntry& ie) {
508 : const IndexEntry& ie_const = ie;
509 : auto oldID = fnGet(nt, ie);
510 : auto newID = getID(oldID, ie_const);
511 : bool b1 = (oldID != newID);
512 : idNumberT newBlack = 0;
513 : bool b2 = (nt == NAME_PLAYER);
514 : if (b2) {
515 : auto oldBlack = ie.GetBlack();
516 : newBlack = getID(oldBlack, ie_const);
517 : b2 = (oldBlack != newBlack);
518 : }
519 : if (!b1 && !b2)
520 : return false;
521 :
522 : if (b1)
523 : fnSet(nt, ie, newID);
524 : if (b2)
525 : ie.SetBlack(newBlack);
526 : return true;
527 : });
528 :
529 : auto err = endTransaction();
530 : res.first = (res.first == OK) ? err : res.first;
531 : return res;
532 : }
533 :
534 : #endif
|
