indico/MaKaC/common/indexes.py

   1 # -*- coding: utf-8 -*-
   2 ##
   3 ##
   4 ## This file is part of CDS Indico.
   5 ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
   6 ##
   7 ## CDS Indico is free software; you can redistribute it and/or
   8 ## modify it under the terms of the GNU General Public License as
   9 ## published by the Free Software Foundation; either version 2 of the
  10 ## License, or (at your option) any later version.
  11 ##
  12 ## CDS Indico is distributed in the hope that it will be useful, but
  13 ## WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ## General Public License for more details.
  16 ##
  17 ## You should have received a copy of the GNU General Public License
  18 ## along with CDS Indico; if not, write to the Free Software Foundation, Inc.,
  19 ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
  20
  21 """This file contains various indexes which will be used in the system in order
  22     to optimise some functionalities.
  23 """
  24 from persistent import Persistent
  25 from BTrees.IOBTree import IOBTree
  26 from BTrees.OOBTree import OOBTree, OOSet
  27 from MaKaC.common.ObjectHolders import ObjectHolder
  28 from MaKaC.common.Configuration import Config
  29 from MaKaC.common.timezoneUtils import nowutc, date2utctimestamp, datetimeToUnixTime
  30 from MaKaC.errors import MaKaCError
  31 from datetime import datetime, timedelta
  32 from MaKaC.i18n import _
  33 from pytz import timezone
  34 from MaKaC.common.logger import Logger
  35 from MaKaC.plugins.base import PluginsHolder
  36 from zope.index.text import textindex
  37 import pytz
  38 import itertools
  39
  40 # BTrees are 32 bit by default
  41 # TODO: make this configurable
  42 # 0111 111 .... max signed int
  43 BTREE_MAX_INT = 0x7FFFFFFF
  44
  45
  46 class Index(Persistent):
  47     _name = ""
  48
  49     def __init__( self, name='' ):
  50         if name != '':
  51             self._name = name
  52         self._words = {}
  53
  54     def getLength( self ):
  55         """ Length of an index.
  56
  57         May be extremely slow for big indexes stored as persistent objects.
  58         Do not use unless you really have to!
  59
  60         """
  61         return len(self._words.keys())
  62
  63     def getKeys( self ):
  64         return self._words.keys()
  65
  66     def getBrowseIndex( self ):
  67         letters = []
  68         words = self.getKeys()
  69         for word in words:
  70             if not word[0].lower() in letters:
  71                 letters.append(word[0].lower())
  72         letters.sort()
  73         return letters
  74
  75     def _addItem( self, value, item ):
  76         if value != "":
  77             words = self._words
  78             if words.has_key(value):
  79                 if item not in words[value]:
  80                     words[value].append(item)
  81             else:
  82                 words[value] = [ item ]
  83             self.setIndex(words)
  84
  85     def _withdrawItem( self, value, item ):
  86         if self._words.has_key(value):
  87             if item in self._words[value]:
  88                 words = self._words
  89                 words[value].remove(item)
  90                 self.setIndex(words)
  91
  92     def matchFirstLetter( self, letter ):
  93         result = []
  94         for key in self.getKeys():
  95             if key[0].lower() == letter.lower():
  96                 result += self._words[key]
  97         return result
  98
  99     def _match( self, value, cs=1, exact=1 ):
 100
 101         result = []
 102         lowerCaseValue = value.lower()
 103
 104         if exact == 1 and cs == 1:
 105             if self._words.has_key(value) and len(self._words[value]) != 0:
 106                 if '' in self._words[value]:
 107                     self._words[value].remove('')
 108                 return self._words[value]
 109             else:
 110                 return None
 111         elif exact == 1 and cs == 0:
 112             for key in self._words.keys():
 113                 if key.lower() == lowerCaseValue and len(self._words[key]) != 0:
 114                     if '' in self._words[key]:
 115                         self._words[key].remove('')
 116                     result = result + self._words[key]
 117             return result
 118         elif exact == 0 and cs == 1:
 119             for key in self._words.keys():
 120                 if key.find(value) != -1 and len(self._words[key]) != 0:
 121                     if '' in self._words[key]:
 122                         self._words[key].remove('')
 123                     result = result + self._words[key]
 124             return result
 125         else:
 126             for key in self._words.keys():
 127                 if key.lower().find(lowerCaseValue) != -1 and len(self._words[key]) != 0:
 128                     if '' in self._words[key]:
 129                         self._words[key].remove('')
 130                     result = result + self._words[key]
 131             return result
 132         return None
 133
 134     def dump(self):
 135         return self._words
 136
 137     def setIndex(self, words):
 138         self._words = words
 139
 140     def notifyModification(self):
 141         self._p_changed=1
 142
 143 class EmailIndex( Index ):
 144     _name = "email"
 145
 146     def indexUser( self, user ):
 147         email = user.getEmail()
 148         self._addItem( email, user.getId() )
 149         for email in user.getSecondaryEmails():
 150             self._addItem( email, user.getId() )
 151
 152     def unindexUser( self, user ):
 153         email = user.getEmail()
 154         self._withdrawItem( email, user.getId() )
 155         for email in user.getSecondaryEmails():
 156             self._withdrawItem( email, user.getId() )
 157
 158     def matchUser( self, email, cs=0, exact=0 ):
 159         """this match is an approximative case insensitive match"""
 160         return self._match(email,cs,exact)
 161
 162 class NameIndex( Index ):
 163     _name = "name"
 164
 165     def indexUser( self, user ):
 166         name = user.getName()
 167         self._addItem( name, user.getId() )
 168
 169     def unindexUser( self, user ):
 170         name = user.getName()
 171         self._withdrawItem( name, user.getId() )
 172
 173     def matchUser( self, name, cs=0, exact=0 ):
 174         """this match is an approximative case insensitive match"""
 175         return self._match(name,cs,exact)
 176
 177 class SurNameIndex( Index ):
 178     _name = "surName"
 179
 180     def indexUser( self, user ):
 181         surName = user.getSurName()
 182         self._addItem( surName, user.getId() )
 183
 184     def unindexUser( self, user ):
 185         surName = user.getSurName()
 186         self._withdrawItem( surName, user.getId() )
 187
 188     def matchUser( self, surName, cs=0, exact=0 ):
 189         """this match is an approximative case insensitive match"""
 190         return self._match(surName,cs,exact)
 191
 192 class OrganisationIndex( Index ):
 193     _name = "organisation"
 194
 195     def indexUser( self, user ):
 196         org = user.getOrganisation()
 197         self._addItem( org, user.getId() )
 198
 199     def unindexUser( self, user ):
 200         org = user.getOrganisation()
 201         self._withdrawItem( org, user.getId() )
 202
 203     def matchUser( self, org, cs=0, exact=0 ):
 204         """this match is an approximative case insensitive match"""
 205         return self._match(org,cs,exact)
 206
 207 class StatusIndex( Index ):
 208     _name = "status"
 209
 210     def __init__( self ):
 211         Index.__init__( self )
 212         from MaKaC.user import AvatarHolder
 213         ah = AvatarHolder()
 214         for av in ah.getList():
 215             self.indexUser(av)
 216
 217     def indexUser( self, user ):
 218         status = user.getStatus()
 219         self._addItem( status, user.getId() )
 220
 221     def unindexUser( self, user ):
 222         status = user.getStatus()
 223         self._withdrawItem( status, user.getId() )
 224
 225     def matchUser( self, status, cs=0, exact=1 ):
 226         """this match is an approximative case insensitive match"""
 227         return self._match(status,cs,exact)
 228
 229 class GroupIndex( Index ):
 230     _name = "group"
 231
 232     def indexGroup( self, group ):
 233         name = group.getName()
 234         self._addItem( name, group.getId() )
 235
 236     def unindexGroup( self, group ):
 237         name = group.getName()
 238         self._withdrawItem( name, group.getId() )
 239
 240     def matchGroup( self, name, cs=0, exact=0 ):
 241         if name == "":
 242             return []
 243         return self._match(name,cs,exact)
 244
 245 class CategoryIndex(Persistent):
 246
 247     def __init__( self ):
 248         self._idxCategItem = OOBTree()
 249
 250     def dump(self):
 251         return list(self._idxCategItem.items())
 252
 253     def _indexConfById(self, categid, confid):
 254         # only the more restrictive setup is taken into account
 255         categid = str(categid)
 256         if self._idxCategItem.has_key(categid):
 257             res = self._idxCategItem[categid]
 258         else:
 259             res = []
 260         res.append(confid)
 261         self._idxCategItem[categid] = res
 262
 263     def unindexConf(self, conf):
 264         confid = str(conf.getId())
 265         self.unindexConfById(confid)
 266
 267     def unindexConfById(self, confid):
 268         for categid in self._idxCategItem.keys():
 269             if confid in self._idxCategItem[categid]:
 270                 res = self._idxCategItem[categid]
 271                 res.remove(confid)
 272                 self._idxCategItem[categid] = res
 273
 274     def reindexCateg(self, categ):
 275         for subcat in categ.getSubCategoryList():
 276             self.reindexCateg(subcat)
 277         for conf in categ.getConferenceList():
 278             self.reindexConf(conf)
 279
 280     def reindexConf(self, conf):
 281         self.unindexConf(conf)
 282         self.indexConf(conf)
 283
 284     def indexConf(self, conf):
 285         categs = conf.getOwnerPath()
 286         level = 0
 287         for categ in conf.getOwnerPath():
 288             if conf.getFullVisibility() > level:
 289                 self._indexConfById(categ.getId(),conf.getId())
 290             level+=1
 291         if conf.getFullVisibility() > level:
 292             self._indexConfById("0",conf.getId())
 293
 294     def getItems(self, categid):
 295         categid = str(categid)
 296         if self._idxCategItem.has_key(categid):
 297             return self._idxCategItem[categid]
 298         else:
 299             return []
 300
 301     def _check(self, dbi=None):
 302         """
 303         Performs some sanity checks
 304         """
 305         i = 0
 306         from MaKaC.conference import ConferenceHolder
 307         confIdx = ConferenceHolder()._getIdx()
 308
 309         for cid, confs in self._idxCategItem.iteritems():
 310             for confId in confs:
 311                 # it has to be in the conference holder
 312                 if confId not in confIdx:
 313                     yield "[%s] '%s' not in ConferenceHolder" % (cid, confId)
 314                 # the category has to be one of the owners
 315                 elif cid not in (map(lambda x:x.id, ConferenceHolder().getById(confId).getOwnerPath()) + ['0']):
 316                     yield "[%s] Conference '%s' is not owned" % (cid, confId)
 317             if dbi and i % 100 == 99:
 318                 dbi.sync()
 319             i += 1
 320
 321
 322 class CalendarIndex(Persistent):
 323     """Implements a persistent calendar-like index. Objects (need to implement
 324         the __hash__ method) with a starting and ending dates can be added to
 325         the index so it's faster and esaier to perform queries in order to
 326         fetch objects which are happening wihtin a given date. Indexing and
 327         unindexing of objects must be explicitely notified by index handlers
 328         i.e. these objects won't be updated when the starting or ending dates
 329         are changed.
 330
 331        Atttributes:
 332         idxSDate - (IOBTree) Inverted index containing all the objects which
 333             are starting on a certain date.
 334         idxEDate - (IOBTree) Inverted index containing all the objects which
 335             are ending on a certain date.
 336     """
 337     def __init__( self ):
 338         self._idxSdate = IOBTree()
 339         self._idxEdate = IOBTree()
 340
 341     def getIdxSdate( self ):
 342         return self._idxSdate
 343
 344     def getIdxEdate( self ):
 345         return self._idxEdate
 346
 347     def dump(self):
 348         return list(self._idxSdate.items())+list(self._idxEdate.items())
 349
 350     def indexConf(self, conf):
 351         """Adds an object to the index for the specified starting and ending
 352             dates.
 353
 354            Parameters:
 355             conf - (Conference) Object to be indexed.
 356         """
 357         confid   = conf.getId()
 358         sdate = date2utctimestamp(conf.getStartDate())
 359         edate = date2utctimestamp(conf.getEndDate())
 360         #checking if 2038 problem occurs
 361         if sdate > BTREE_MAX_INT or edate > BTREE_MAX_INT:
 362             return
 363         if self._idxSdate.has_key( sdate ):
 364             res = self._idxSdate[sdate]
 365         else:
 366             res = []
 367         if not confid in res:
 368             res.append(confid)
 369             self._idxSdate[sdate] = res
 370         if self._idxEdate.has_key( edate ):
 371             res = self._idxEdate[edate]
 372         else:
 373             res = []
 374         if not confid in res:
 375             res.append(confid)
 376             self._idxEdate[edate] = res
 377
 378     def unindexConf( self, conf):
 379         """Removes the given object from the index for the specified starting
 380             and ending dates. If the object is not indexed at any of the
 381             2 values the operation won't be executed.
 382
 383            Raises:
 384             Exception - the specified object is not indexed at the specified
 385                     dates
 386
 387            Parameters:
 388             conf - (Conference) Object to be removed from the index.
 389             sdate - (datetime) Starting date at which the object is indexed.
 390             edate - (datetime) [optional] Ending date at which the object
 391                 is indexed. If it is not specified the starting date will be
 392                 used as ending one.
 393         """
 394         confid = conf.getId()
 395         sdate = date2utctimestamp(conf.getStartDate())
 396         edate = date2utctimestamp(conf.getEndDate())
 397         #checking if 2038 problem occurs
 398         if sdate > BTREE_MAX_INT or edate > BTREE_MAX_INT:
 399             return
 400         if not self._idxSdate.has_key( sdate ):
 401             for key in self._idxSdate.keys():
 402                 res = self._idxSdate[key]
 403                 while confid in res:
 404                     res.remove(confid)
 405                     self._idxSdate[key] = res
 406                     if len(self._idxSdate[key]) == 0:
 407                         del self._idxSdate[key]
 408         else:
 409             while confid in self._idxSdate[sdate]:
 410                 res = self._idxSdate[sdate]
 411                 res.remove(confid)
 412                 self._idxSdate[sdate] = res
 413             if len(self._idxSdate[sdate]) == 0:
 414                 del self._idxSdate[sdate]
 415         if not self._idxEdate.has_key( edate ):
 416             for key in self._idxEdate.keys():
 417                 res = self._idxEdate[key]
 418                 while confid in res:
 419                     res.remove(confid)
 420                     self._idxEdate[key] = res
 421                     if len(self._idxEdate[key]) == 0:
 422                         del self._idxEdate[key]
 423         else:
 424             while confid in self._idxEdate[edate]:
 425                 res = self._idxEdate[edate]
 426                 res.remove(confid)
 427                 self._idxEdate[edate] = res
 428             if len(self._idxEdate[edate]) == 0:
 429                 del self._idxEdate[edate]
 430
 431     def _getObjectsStartingBefore( self, date ):
 432         """Returns all the objects starting before the specified day (excluded).
 433
 434            Parameters:
 435             date - (tz aware datetime) Date for which all the indexed objects starting
 436                 before have to be fecthed and returned.
 437            Returns:
 438             (Set) - set of objects starting before the specified day
 439         """
 440         date = date2utctimestamp(date)
 441         res = set()
 442         for val in self._idxSdate.values(self._idxSdate.minKey(), date):
 443             res.update( set( val ) )
 444         return res
 445
 446     def _getObjectsStartingAfter( self, date ):
 447         """Returns all the objects starting after the specified day (excluded).
 448
 449            Parameters:
 450             date - (tz aware datetime) Date for which all the indexed objects starting
 451                 after have to be fecthed and returned.
 452            Returns:
 453             (Set) - set of objects starting before the specified day
 454         """
 455         date = date2utctimestamp(date)
 456         res = set()
 457         for val in self._idxSdate.values(date):
 458             res.update( set( val ) )
 459         return res
 460
 461     def _getObjectsEndingBefore( self, date ):
 462         """Returns all the objects ending before the specified day (excluded).
 463
 464            Parameters:
 465             date - (tz aware datetime) Date for which all the indexed objects ending
 466                 before have to be fecthed and returned.
 467            Returns:
 468             (Set) - set of objects ending before the specified day
 469         """
 470         date = date2utctimestamp(date)
 471         res = set()
 472         for val in self._idxEdate.values(self._idxEdate.minKey(), date):
 473             res.update( set( val ) )
 474         return res
 475
 476     def _getObjectsEndingAfter( self, date ):
 477         """Returns all the objects ending after the specified day (included).
 478
 479            Parameters:
 480             date - (tz aware datetime) Date for which all the indexed objects ending
 481                 after have to be fecthed and returned.
 482            Returns:
 483             (Set) - set of objects ending after the specified day
 484         """
 485         date = date2utctimestamp(date)
 486         res = set()
 487         for val in self._idxEdate.values(date):
 488             res.update( set( val ) )
 489         return res
 490
 491     def getObjectsStartingInDay( self, date ):
 492         """Returns all the objects which are starting on the specified date.
 493
 494            Parameters:
 495             date - (tz aware datetime) day
 496
 497            Returns:
 498             (Set) - set of objects happening within the specified date interval.
 499         """
 500         sDate = date.replace(hour=0, minute=0, second=0, microsecond=0)
 501         eDate = date.replace(hour=23, minute=59, second=59, microsecond=0)
 502         return self.getObjectsStartingIn(sDate,eDate)
 503
 504     def getObjectsInDay( self, date ):
 505         """Returns all the objects which are happening on the specified date.
 506
 507            Parameters:
 508             date - (tz aware datetime) day
 509
 510            Returns:
 511             (Set) - set of objects happening within the specified date interval.
 512         """
 513         sDate = date.replace(hour=0, minute=0, second=0, microsecond=0)
 514         eDate = date.replace(hour=23, minute=59, second=59, microsecond=0)
 515         return self.getObjectsIn(sDate,eDate)
 516
 517     def getObjectsStartingIn( self, sDate, eDate):
 518         """Returns all the objects which are starting whithin the specified
 519             date interval.
 520
 521            Parameters:
 522             sDate - (tz aware datetime) Lower date of the interval to be considered.
 523             eDate - (tz aware datetime) Higher date of the interval to be considered.
 524
 525            Returns:
 526             (Set) - set of objects happening within the specified date interval.
 527         """
 528         s2 = self._getObjectsStartingAfter( sDate )
 529         if not s2:
 530             return s2
 531         s1 = self._getObjectsStartingBefore( eDate )
 532         s2.intersection_update( s1 )
 533         return s2
 534
 535     def getObjectsEndingIn( self, sDate, eDate):
 536         """Returns all the objects which are ending whithin the specified
 537             date interval.
 538            Parameters:
 539             sDate - (tz aware datetime) Lower date of the interval to be considered.
 540             eDate - (tz aware datetime) Higher date of the interval to be considered.
 541            Returns:
 542             (Set) - set of objects ending within the specified date interval.
 543         """
 544         s2 = self._getObjectsEndingAfter( sDate )
 545         if not s2:
 546             return s2
 547         s1 = self._getObjectsEndingBefore( eDate )
 548         s2.intersection_update( s1 )
 549         return s2
 550
 551     def getObjectsEndingInDay( self, date ):
 552         """Returns all the objects which are ending on the specified date.
 553
 554            Parameters:
 555             date - (tz aware datetime) day
 556
 557            Returns:
 558             (Set) - set of objects happening within the specified date interval.
 559         """
 560         sDate = date.replace(hour=0, minute=0, second=0, microsecond=0)
 561         eDate = date.replace(hour=23, minute=59, second=59, microsecond=0)
 562         return self.getObjectsEndingIn(sDate,eDate)
 563
 564     def getObjectsIn( self, sDate, eDate ):
 565         """Returns all the objects which are happening whithin the specified
 566             date interval.
 567
 568            Parameters:
 569             sDate - (tz aware datetime) Lower date of the interval to be considered.
 570             eDate - (tz aware datetime) Higher date of the interval to be considered.
 571
 572            Returns:
 573             (Set) - set of objects happening within the specified date interval.
 574         """
 575         s2 = self._getObjectsEndingAfter( sDate )
 576         if not s2:
 577             return s2
 578         s1 = self._getObjectsStartingBefore( eDate )
 579         s2.intersection_update( s1 )
 580         return s2
 581
 582     def getObjectsEndingAfter( self, date ):
 583         return self._getObjectsEndingAfter( date )
 584
 585     def getObjectsStartingAfter( self, date ):
 586         return self._getObjectsStartingAfter( date )
 587
 588     def _check(self, dbi=None):
 589         """
 590         Performs some sanity checks
 591         """
 592
 593         from MaKaC.conference import ConferenceHolder
 594         confIdx = ConferenceHolder()._getIdx()
 595
 596         def _check_index(desc, index, func):
 597             i = 0
 598             for ts, confs in index.iteritems():
 599                 for confId in confs:
 600                     # it has to be in the conference holder
 601                     if confId not in confIdx:
 602                         yield "[%s][%s] '%s' not in ConferenceHolder" % (desc, ts, confId)
 603                     else:
 604
 605                         conf = ConferenceHolder().getById(confId)
 606                         try:
 607                             expectedDate = date2utctimestamp(func(conf))
 608                         except OverflowError:
 609                             expectedDate = 'overflow'
 610
 611                         # ts must be ok
 612                         if ts != expectedDate:
 613                             yield "[%s][%s] Conference '%s' has bogus date (should be '%s')" % (desc, ts, confId, expectedDate)
 614                     if dbi and i % 100 == 99:
 615                         dbi.sync()
 616                     i += 1
 617
 618         return itertools.chain(
 619             _check_index('sdate', self._idxSdate, lambda x: x.getStartDate()),
 620             _check_index('edate', self._idxEdate, lambda x: x.getEndDate()))
 621
 622
 623 class CalendarDayIndex(Persistent):
 624     def __init__( self ):
 625         self._idxDay = IOBTree()
 626
 627     def getIdxDate( self ):
 628         return self._idxDay
 629
 630     def dump(self):
 631         return list(self._idxDay.items())
 632
 633     def indexConf(self, conf):
 634         # Note: conf can be any object which has getEndDate() and getStartDate() methods
 635         self._idxDay._p_changed = True
 636         days = (conf.getEndDate().date() - conf.getStartDate().date()).days
 637         startDate = datetime(conf.getStartDate().year, conf.getStartDate().month, conf.getStartDate().day)
 638         for day in range(days + 1):
 639             key = int(datetimeToUnixTime(startDate + timedelta(day)))
 640             #checking if 2038 problem occurs
 641             if key > BTREE_MAX_INT:
 642                 continue
 643             if self._idxDay.has_key(key):
 644                 self._idxDay[key].add(conf)
 645             else:
 646                 self._idxDay[key] = OOSet([conf])
 647
 648
 649     def unindexConf( self, conf):
 650         # Note: conf can be any object which has getEndDate() and getStartDate() methods
 651         self._idxDay._p_changed = True
 652         days = (conf.getEndDate().date() - conf.getStartDate().date()).days
 653         startDate = datetime(conf.getStartDate().year, conf.getStartDate().month, conf.getStartDate().day)
 654         for dayNumber in range(days + 1):
 655             day = int(datetimeToUnixTime(startDate + timedelta(dayNumber)))
 656             #checking if 2038 problem occurs
 657             if day > BTREE_MAX_INT:
 658                 continue
 659             if self._idxDay.has_key( day ):
 660                 if conf in self._idxDay[day]:
 661                     self._idxDay[day].remove(conf)
 662                 if len(self._idxDay[day]) == 0:
 663                     del self._idxDay[day]
 664
 665
 666     def getObjectsStartingInDay( self, date ):
 667         day = datetime(date.year, date.month, date.day, tzinfo = date.tzinfo)
 668         if self._idxDay.has_key(int(datetimeToUnixTime(day))):
 669             return set([event for event in self._idxDay[int(datetimeToUnixTime(day))] if event.getStartDate() >= day])
 670         else:
 671             return set()
 672
 673     def getObjectsEndingInDay( self, date ):
 674         day = datetime(date.year, date.month, date.day, tzinfo = date.tzinfo)
 675         if self._idxDay.has_key(int(datetimeToUnixTime(day))):
 676             return set([event for event in self._idxDay[int(datetimeToUnixTime(day))] if event.getEndDate() < day + timedelta(1)])
 677         else:
 678             return set()
 679
 680     def getObjectsInDay( self, date ):
 681         day = datetime(date.year, date.month, date.day)
 682         if self._idxDay.has_key(int(datetimeToUnixTime(day))):
 683             return set(self._idxDay[int(datetimeToUnixTime(day))])
 684         else:
 685             return set()
 686
 687     def getObjectsStartingIn( self, sDate, eDate):
 688         sDay = datetime(sDate.year, sDate.month, sDate.day, tzinfo = sDate.tzinfo)
 689         eDay = datetime(eDate.year, eDate.month, eDate.day, tzinfo = eDate.tzinfo)
 690         res = set()
 691         if sDay == eDay:
 692             if self._idxDay.has_key(int(datetimeToUnixTime(sDay))):
 693                 res = set([event for event in self._idxDay[int(datetimeToUnixTime(sDay))] if event.getStartDate() <= eDate and event.getStartDate() >= sDate])
 694         elif sDay < eDay:
 695             if self._idxDay.has_key(int(datetimeToUnixTime(sDay))):
 696                 res = set([event for event in self._idxDay[int(datetimeToUnixTime(sDay))] if event.getStartDate() >= sDate])
 697             if self._idxDay.has_key(int(datetimeToUnixTime(eDay))):
 698                 res.update([event for event in self._idxDay[int(datetimeToUnixTime(eDay))] if event.getStartDate() <= eDate and event.getStartDate() >= eDay ])
 699             for day in range((eDay - sDay).days - 1):
 700                 res.update(self.getObjectsStartingInDay( sDay + timedelta(1 + day)))
 701         return res
 702
 703     def getObjectsEndingIn( self, sDate, eDate):
 704         sDay = datetime(sDate.year, sDate.month, sDate.day, tzinfo = sDate.tzinfo)
 705         eDay = datetime(eDate.year, eDate.month, eDate.day, tzinfo = eDate.tzinfo)
 706         res = set()
 707         if sDay == eDay:
 708             if self._idxDay.has_key(int(datetimeToUnixTime(sDay))):
 709                 res = set([event for event in self._idxDay[int(datetimeToUnixTime(sDay))] if event.getEndDate() <= eDate and event.getEndDate() >= sDate])
 710         elif sDay < eDay:
 711             res = set()
 712             if self._idxDay.has_key(int(datetimeToUnixTime(sDay))):
 713                 res = set([event for event in self._idxDay[int(datetimeToUnixTime(sDay))] if event.getEndDate() >= sDate and event.getEndDate() < sDay + timedelta(1)])
 714             if self._idxDay.has_key(int(datetimeToUnixTime(eDay))):
 715                 res.update([event for event in self._idxDay[int(datetimeToUnixTime(eDay))] if event.getEndDate() <= eDate])
 716             for day in range((eDay - sDay).days - 1):
 717                 res.update(self.getObjectsEndingInDay( sDay + timedelta(1 + day)))
 718         return res
 719
 720     def getObjectsIn( self, sDate, eDate ):
 721         """
 722         TODO: Reimplement using iterateObjectsIn!!
 723         (or get rid of this one, as the other should be faster)
 724         """
 725
 726         sDay = datetime(sDate.year, sDate.month, sDate.day)
 727         eDay = datetime(eDate.year, eDate.month, eDate.day)
 728         res = set()
 729         if sDay == eDay:
 730             if self._idxDay.has_key(int(datetimeToUnixTime(sDay))):
 731                 res = set([event for event in self._idxDay[int(datetimeToUnixTime(sDay))] if event.getStartDate() <= eDate and event.getEndDate() >= sDate])
 732         elif sDay < eDay:
 733             res = set()
 734             if self._idxDay.has_key(int(datetimeToUnixTime(sDay))):
 735                 res = set([event for event in self._idxDay[int(datetimeToUnixTime(sDay))] if event.getEndDate() >= sDate])
 736             if self._idxDay.has_key(int(datetimeToUnixTime(eDay))):
 737                 res.update([event for event in self._idxDay[int(datetimeToUnixTime(eDay))] if event.getStartDate() <= eDate])
 738             res.update(self.getObjectsInDays( sDay + timedelta(1), eDay - timedelta(1) ))
 739         return res
 740
 741     def iterateObjectsIn(self, sDate, eDate):
 742         """
 743         Returns all the events between two dates taking into account the starting and ending times.
 744         """
 745         sDay = datetime(sDate.year, sDate.month, sDate.day) if sDate else None
 746         eDay = datetime(eDate.year, eDate.month, eDate.day) if eDate else None
 747
 748         if sDay and sDay == eDay:
 749             if int(datetimeToUnixTime(sDay)) in self._idxDay:
 750                 for event in self._idxDay[int(datetimeToUnixTime(sDay))]:
 751                     if event.getStartDate() <= eDate and event.getEndDate() >= sDate:
 752                         yield event
 753             return
 754
 755         if sDay and int(datetimeToUnixTime(sDay)) in self._idxDay:
 756             for event in self._idxDay[int(datetimeToUnixTime(sDay))]:
 757                 if event.getEndDate() >= sDate:
 758                     yield event
 759
 760         if sDay and eDay:
 761             fromTS, toTS = sDay + timedelta(1), eDay - timedelta(1)
 762         elif sDay:
 763             fromTS, toTS = sDay + timedelta(), None
 764         elif eDay:
 765             fromTS, toTS = None, eDay - timedelta(1)
 766         else:
 767             fromTS, toTS = None, None
 768
 769         for evt in self.iterateObjectsInDays(fromTS, toTS):
 770             yield evt
 771
 772         if eDay and int(datetimeToUnixTime(eDay)) in self._idxDay:
 773             for event in self._idxDay[int(datetimeToUnixTime(eDay))]:
 774                 if event.getStartDate() <= eDate:
 775                     yield event
 776
 777     def getObjectsInDays( self, sDate=None, eDate=None ):
 778         sDay = int(datetimeToUnixTime(datetime(sDate.year, sDate.month, sDate.day))) if sDate else None
 779         eDay = int(datetimeToUnixTime(datetime(eDate.year, eDate.month, eDate.day))) if eDate else None
 780         res = set()
 781         for event in self._idxDay.values(sDay, eDay):
 782             res.update(event)
 783         return res
 784
 785     def iterateObjectsInDays(self, sDate=None, eDate=None):
 786         """
 787         Returns all the events between two dates WITHOUT taking into account the starting and ending times.
 788         """
 789
 790         sDay = int(datetimeToUnixTime(datetime(sDate.year, sDate.month, sDate.day))) if sDate else None
 791         eDay = int(datetimeToUnixTime(datetime(eDate.year, eDate.month, eDate.day))) if eDate else None
 792         for day in self._idxDay.itervalues(sDay, eDay):
 793             for event in day:
 794                 yield event
 795
 796     def getObjectsEndingAfter( self, date ):
 797         day = datetime(date.year, date.month, date.day)
 798         nextDay = day + timedelta(1)
 799         res = set()
 800         if self._idxDay.has_key(int(datetimeToUnixTime(day))):
 801             res = set([event for event in self._idxDay[int(datetimeToUnixTime(day))] if event.getEndDate() >= date])
 802         for day in self._idxDay.values(int(datetimeToUnixTime(nextDay))):
 803             res.update(set(day))
 804         return res
 805
 806     def getObjectsStartingAfter( self, date ):
 807         stDay = datetime(date.year, date.month, date.day)
 808         nextDay = stDay + timedelta(1)
 809         previousDay = stDay - timedelta(1)
 810         res = set()
 811         if self._idxDay.has_key(int(datetimeToUnixTime(stDay))):
 812             res = set([event for event in self._idxDay[int(datetimeToUnixTime(stDay))] if event.getStartDate() >= date])
 813         for day in self._idxDay.values(int(datetimeToUnixTime(nextDay))):
 814             res.update(set(day))
 815         for day in self._idxDay.values(max = int(datetimeToUnixTime(previousDay))):
 816             res.difference_update(set(day))
 817         res.difference_update(set([event for event in self._idxDay[int(datetimeToUnixTime(stDay))] if event.getStartDate() < date]))
 818         return res
 819
 820     def iterateObjectsIn(self, sDate, eDate):
 821         sDay = datetime(sDate.year, sDate.month, sDate.day) if sDate else None
 822         eDay = datetime(eDate.year, eDate.month, eDate.day) if eDate else None
 823
 824         if sDay and sDay == eDay:
 825             if int(datetimeToUnixTime(sDay)) in self._idxDay:
 826                 for event in self._idxDay[int(datetimeToUnixTime(sDay))]:
 827                     if event.getStartDate() <= eDate and event.getEndDate() >= sDate:
 828                         yield event
 829             return
 830
 831         # keep track of the records that have been already sent
 832
 833         if sDay and int(datetimeToUnixTime(sDay)) in self._idxDay:
 834             for event in self._idxDay[int(datetimeToUnixTime(sDay))]:
 835                 if event.getEndDate() >= sDate:
 836                     yield event
 837
 838         if sDay and eDay:
 839             fromTS, toTS = sDay + timedelta(1), eDay - timedelta(1)
 840         elif sDay:
 841             fromTS, toTS = sDay + timedelta(), None
 842         elif eDay:
 843             fromTS, toTS = None, eDay - timedelta(1)
 844         else:
 845             fromTS, toTS = None, None
 846
 847         for evt in self.iterateObjectsInDays(fromTS, toTS):
 848             yield evt
 849
 850         if eDay and int(datetimeToUnixTime(eDay)) in self._idxDay:
 851             for event in self._idxDay[int(datetimeToUnixTime(eDay))]:
 852                 if event.getStartDate() <= eDate:
 853                     yield event
 854
 855     def iterateObjectsInDays(self, sDate=None, eDate=None):
 856
 857         sDay = int(datetimeToUnixTime(datetime(sDate.year, sDate.month, sDate.day))) if sDate else None
 858         eDay = int(datetimeToUnixTime(datetime(eDate.year, eDate.month, eDate.day))) if eDate else None
 859         for day in self._idxDay.itervalues(sDay, eDay):
 860             for event in day:
 861                 yield event
 862
 863     def _check(self, dbi=None, categId=''):
 864         """
 865         Performs some sanity checks
 866         """
 867         i = 0
 868         from MaKaC.conference import ConferenceHolder
 869         confIdx = ConferenceHolder()._getIdx()
 870
 871         for ts, confs in self._idxDay.iteritems():
 872             dt = pytz.timezone('UTC').localize(datetime.utcfromtimestamp(ts))
 873
 874             for conf in confs:
 875                 # it has to be in the conference holder
 876                 if conf.getId() not in confIdx:
 877                     yield "[%s][%s] '%s' not in ConferenceHolder" % (ts, categId, conf.getId())
 878                 else:
 879                     # date must be ok
 880                     if dt > conf.getEndDate().replace(hour=23, minute=59, second=59) \
 881                            or dt < conf.getStartDate().replace(hour=0, minute=0, second=0):
 882                         yield "[%s] '%s' has date out of bounds '%s'(%s)" % (categId, conf.getId(), ts, dt)
 883                     elif categId not in (map(lambda x:x.id, conf.getOwnerPath()) + ['0']):
 884                         yield "[%s] Conference '%s' is not owned" % (categId, conf.getId())
 885
 886                 if dbi and i % 100 == 99:
 887                     dbi.sync()
 888                 i += 1
 889
 890
 891 class CategoryDateIndex(Persistent):
 892
 893     def __init__( self ):
 894         self._idxCategItem = OOBTree()
 895
 896     def dump(self):
 897         return map(lambda idx: (idx[0], idx[1].dump()), list(self._idxCategItem.items()))
 898
 899     def unindexConf(self, conf):
 900         for owner in conf.getOwnerPath():
 901             if self._idxCategItem.has_key(owner.getId()):
 902                 self._idxCategItem[owner.getId()].unindexConf(conf)
 903         if self._idxCategItem.has_key('0'):
 904             self._idxCategItem['0'].unindexConf(conf)
 905
 906
 907     def reindexCateg(self, categ):
 908         for subcat in categ.getSubCategoryList():
 909             self.reindexCateg(subcat)
 910         for conf in categ.getConferenceList():
 911             self.unindexConf(conf)
 912             self.indexConf(conf)
 913 #        from MaKaC.common import DBMgr
 914 #        dbi = DBMgr.getInstance()
 915 #        for subcat in categ.getSubCategoryList():
 916 #            self.reindexCateg(subcat)
 917 #        for conf in categ.getConferenceList():
 918 #            while True:
 919 #                try:
 920 #                    dbi.sync()
 921 #                    self.unindexConf(conf)
 922 #                    self.indexConf(conf)
 923 #                    dbi.commit()
 924 #                    break
 925 #                except:
 926 #                    print 'Exception commiting conf %s'%conf.getId()
 927
 928     def unindexCateg(self, categ):
 929         for subcat in categ.getSubCategoryList():
 930             self.unindexCateg(subcat)
 931         for conf in categ.getConferenceList():
 932             self.unindexConf(conf)
 933
 934     def indexCateg(self, categ):
 935         for subcat in categ.getSubCategoryList():
 936             self.indexCateg(subcat)
 937         for conf in categ.getConferenceList():
 938             self.indexConf(conf)
 939
 940     def _indexConf(self, categid, conf):
 941         # only the more restrictive setup is taken into account
 942         if self._idxCategItem.has_key(categid):
 943             res = self._idxCategItem[categid]
 944         else:
 945             res = CalendarIndex()
 946         res.indexConf(conf)
 947         self._idxCategItem[categid] = res
 948
 949     # TOREMOVE?? defined in CategoryDayIndex
 950     def indexConf(self, conf):
 951         for categ in conf.getOwnerPath():
 952             self._indexConf(categ.getId(), conf)
 953         self._indexConf("0",conf)
 954
 955     def getObjectsIn(self, categid, sDate, eDate):
 956         categid = str(categid)
 957         if self._idxCategItem.has_key(categid):
 958             return self._idxCategItem[categid].getObjectsIn(sDate, eDate)
 959         else:
 960             return []
 961
 962     def getObjectsStartingIn( self, categid, sDate, eDate):
 963         categid = str(categid)
 964         if self._idxCategItem.has_key(categid):
 965             return self._idxCategItem[categid].getObjectsStartingIn(sDate, eDate)
 966         else:
 967             return []
 968
 969     def getObjectsInDay( self, categid, sDate):
 970         categid = str(categid)
 971         if self._idxCategItem.has_key(categid):
 972             return self._idxCategItem[categid].getObjectsInDay(sDate)
 973         else:
 974             return []
 975
 976     def getObjectsEndingAfter( self, categid, sDate):
 977         categid = str(categid)
 978         if self._idxCategItem.has_key(categid):
 979             return self._idxCategItem[categid].getObjectsEndingAfter(sDate)
 980         else:
 981             return []
 982
 983 class CategoryDateIndexLtd(CategoryDateIndex):
 984     """ Version of CategoryDateIndex whiself.ch indexing events
 985         on the base of their visibility
 986     """
 987     def indexConf(self, conf):
 988         level = 0
 989         for categ in conf.getOwnerPath():
 990             if conf.getFullVisibility() > level:
 991                 self._indexConf(categ.getId(),conf)
 992             level+=1
 993         if conf.getFullVisibility() > level:
 994             self._indexConf("0",conf)
 995
 996     def buildIndex(self):
 997         self._idxCategItem = OOBTree()
 998         from MaKaC.conference import CategoryManager
 999         self.indexCateg(CategoryManager().getById('0'))
1000
1001 class CategoryDayIndex(CategoryDateIndex):
1002
1003     def __init__(self, visibility=True):
1004         super(CategoryDayIndex, self).__init__()
1005         self._useVisibility = visibility
1006
1007     def _indexConf(self, categid, conf):
1008         # only the more restrictive setup is taken into account
1009         if self._idxCategItem.has_key(categid):
1010             res = self._idxCategItem[categid]
1011         else:
1012             res = CalendarDayIndex()
1013         res.indexConf(conf)
1014         self._idxCategItem[categid] = res
1015
1016     def reindexConf(self, conf):
1017         self.unindexConf(conf)
1018         self.indexConf(conf)
1019
1020     def indexConf(self, conf):
1021         level = 0
1022         for categ in conf.getOwnerPath():
1023             if not self._useVisibility or conf.getFullVisibility() > level:
1024                 self._indexConf(categ.getId(),conf)
1025             level+=1
1026         if not self._useVisibility or conf.getFullVisibility() > level:
1027             self._indexConf("0",conf)
1028
1029     def buildIndex(self):
1030         self._idxCategItem = OOBTree()
1031         from MaKaC.conference import CategoryManager
1032         self.indexCateg(CategoryManager().getById('0'))
1033
1034     def getObjectsInDays(self, categid, sDate, eDate):
1035         if self._idxCategItem.has_key(categid):
1036             return self._idxCategItem[categid].getObjectsInDays(sDate, eDate)
1037         else:
1038             return []
1039
1040     def iterateObjectsIn(self, categid, sDate, eDate):
1041         if categid in self._idxCategItem:
1042             return self._idxCategItem[categid].iterateObjectsIn(sDate, eDate)
1043         else:
1044             return []
1045
1046     def _check(self, dbi=None):
1047         """
1048         Performs some sanity checks
1049         """
1050         for categId, calDayIdx in self._idxCategItem.iteritems():
1051             for problem in calDayIdx._check(dbi=dbi, categId=categId):
1052                 yield problem
1053
1054
1055 class PendingQueuesUsersIndex( Index ):
1056     _name = ""
1057
1058     def indexPendingUser( self, user ):
1059         email = user.getEmail().lower()
1060         self._addItem( email, user )
1061         self.notifyModification()
1062
1063     def unindexPendingUser( self, user ):
1064         email = user.getEmail().lower()
1065         self._withdrawItem( email, user )
1066         self.notifyModification()
1067
1068     def _withdrawItem( self, value, item ):
1069         Index._withdrawItem( self, value, item )
1070         if self._words.has_key(value):
1071             if self._words[value]==[]:
1072                 del self._words[value]
1073
1074     def matchPendingUser( self, email, cs=0, exact=1 ):
1075         """this match is an approximative case insensitive match"""
1076         return self._match(email,cs,exact)
1077
1078 class PendinQueuesTasksIndex( Index ):
1079     _name = ""
1080
1081     def indexTask( self, email, task ):
1082         email = email.lower().strip()
1083         self._addItem( email, task )
1084         self.notifyModification()
1085
1086     def unindexTask( self, email, task ):
1087         email = email.lower().strip()
1088         self._withdrawItem( email, task )
1089         self.notifyModification()
1090
1091     def _withdrawItem( self, value, item ):
1092         Index._withdrawItem( self, value, item )
1093         if self._words.has_key(value):
1094             if self._words[value]==[]:
1095                 del self._words[value]
1096
1097     def matchTask( self, email, cs=0, exact=1 ):
1098         """this match is an approximative case insensitive match"""
1099         return self._match(email,cs,exact)
1100
1101 class PendingSubmittersIndex( PendingQueuesUsersIndex ):
1102     _name = "pendingSubmitters"
1103     pass
1104
1105 class PendingSubmittersTasksIndex( PendinQueuesTasksIndex ):
1106     _name = "pendingSubmittersTasks"
1107     pass
1108
1109 class PendingManagersIndex( PendingQueuesUsersIndex ):
1110     _name = "pendingManagers"
1111     pass
1112
1113 class PendingManagersTasksIndex( PendinQueuesTasksIndex ):
1114     _name = "pendingManagersTasks"
1115     pass
1116
1117 class PendingCoordinatorsIndex( PendingQueuesUsersIndex ):
1118     _name = "pendingCoordinators"
1119     pass
1120
1121 class PendingCoordinatorsTasksIndex( PendinQueuesTasksIndex ):
1122     _name = "pendingCoordinatorsTasks"
1123     pass
1124
1125
1126 class IndexException(Exception):
1127     pass
1128
1129
1130 class IntStringMappedIndex(Persistent):
1131     def __init__(self):
1132         self._intToStrMap = {}
1133         self._strToIntMap = {}
1134         self._counter = 0
1135
1136     def addString(self, stringId):
1137         """
1138         Adds a string to the index, returning the
1139         assigned integer
1140         """
1141
1142         if stringId in self._strToIntMap:
1143             raise KeyError("Key '%s' already exists in index!" % stringId)
1144
1145         intId = self._counter
1146         self._intToStrMap[intId] = stringId
1147         self._strToIntMap[stringId] = intId
1148         self._counter += 1
1149
1150         self._p_changed = 1
1151
1152         return intId
1153
1154     def removeString(self, stringId):
1155         """
1156         Removes an entry from the Int-String Mapped index,
1157         taking the string as input, and returning the integer
1158         if it exists, or -1 otherwise.
1159         """
1160
1161         intId = self._strToIntMap[stringId]
1162         try:
1163             del self._strToIntMap[stringId]
1164             del self._intToStrMap[intId]
1165         except KeyError:
1166             return -1
1167
1168         self._p_changed = 1
1169
1170         return intId
1171
1172     def getString(self, intId):
1173         """
1174         0 -> 'abcd'
1175         """
1176
1177         if type(intId) != int:
1178             raise TypeError
1179         if intId not in self._intToStrMap:
1180             return None
1181
1182         return self._intToStrMap[intId]
1183
1184     def getInteger(self, strId):
1185         """
1186         'abcd' -> 0
1187         """
1188
1189         if type(strId) != str:
1190             raise TypeError
1191         if strId not in self._strToIntMap:
1192             return None
1193
1194         return self._strToIntMap[strId]
1195
1196
1197 class TextIndex(IntStringMappedIndex):
1198
1199     def __init__(self):
1200         IntStringMappedIndex.__init__(self)
1201         self._textIdx = textindex.TextIndex()
1202
1203     def index(self, entryId, title):
1204         intId = self.addString(entryId)
1205         self._textIdx.index_doc(intId, title)
1206
1207     def unindex(self, entryId):
1208         intId = self.getInteger(entryId)
1209
1210         if intId != None:
1211             self.removeString(entryId)
1212             self._textIdx.unindex_doc(intId)
1213         else:
1214             Logger.get('indexes.text').error("No such entry '%s'" % entryId)
1215
1216     def search(self, text):
1217         records = self._textIdx.apply(text.decode('utf8')).items()
1218         return [(self.getString(record[0]), record[1]) for record in records]
1219
1220
1221 class IndexesHolder( ObjectHolder ):
1222
1223     idxName = "indexes"
1224     counterName = None
1225     __allowedIdxs = [ "email", "name", "surName", "organisation", "group",
1226                     "status", "calendar", "category", "categoryDate",
1227                     "categoryDateAll", "categoryName",
1228                     "pendingSubmitters",
1229                     "pendingSubmittersTasks", "pendingManagers",
1230                     "pendingManagersTasks", "pendingCoordinators",
1231                     "pendingCoordinatorsTasks", "webcasts", "collaboration"]
1232
1233     def getIndex( self, name ):
1234         return self.getById(name)
1235
1236     def getById( self, id ):
1237         """returns an object from the index which id corresponds to the one
1238             which is specified.
1239         """
1240
1241         if id not in self.__allowedIdxs:
1242             raise MaKaCError( _("Unknown index: %s")%id)
1243         Idx = self._getIdx()
1244         if id in Idx:
1245             return Idx[id]
1246         else:
1247             if id=="email":
1248                 Idx[str(id)] = EmailIndex()
1249             elif id=="name":
1250                 Idx[str(id)] = NameIndex()
1251             elif id=="surName":
1252                 Idx[str(id)] = SurNameIndex()
1253             elif id=="organisation":
1254                 Idx[str(id)] = OrganisationIndex()
1255             elif id=="status":
1256                 Idx[str(id)] = StatusIndex()
1257             elif id=="group":
1258                 Idx[str(id)] = GroupIndex()
1259             elif id=="calendar":
1260                 Idx[str(id)] = CalendarIndex()
1261             elif id=="category":
1262                 Idx[str(id)] = CategoryIndex()
1263             elif id=="categoryDate":
1264                 Idx[str(id)] = CategoryDayIndex()
1265             elif id=="categoryName":
1266                 Idx[str(id)] = TextIndex()
1267             elif id=="pendingSubmitters":
1268                 Idx[str(id)] = PendingSubmittersIndex()
1269             elif id=="pendingSubmittersTasks":
1270                 Idx[str(id)] = PendingSubmittersTasksIndex()
1271             elif id=="pendingManagers":
1272                 Idx[str(id)] = PendingManagersIndex()
1273             elif id=="pendingManagersTasks":
1274                 Idx[str(id)] = PendingManagersTasksIndex()
1275             elif id=="pendingCoordinators":
1276                 Idx[str(id)] = PendingManagersIndex()
1277             elif id=="pendingCoordinatorsTasks":
1278                 Idx[str(id)] = PendingManagersTasksIndex()
1279             elif id=="collaboration":
1280                 if PluginsHolder().hasPluginType("Collaboration"):
1281                     from MaKaC.plugins.Collaboration.indexes import CollaborationIndex
1282                     Idx[str(id)] = CollaborationIndex()
1283                 else:
1284                     raise MaKaCError(_("Tried to retrieve collaboration index, but Collaboration plugins are not present"))
1285
1286             return Idx[str(id)]
1287
1288
1289 if __name__ == "__main__":
1290     print _("done")
1291
1292
1293