root/trunk/LeMillCatalogTool.py

#
# Plone CatalogTool simplified -- does not use ExtensibleIndexableObjectWrapper, workflows or permissions. Basically just ZCatalog posing as Plone Tool, allowing but ignoring plonish commands.
#

from Products.CMFPlone import ToolNames 
from AccessControl import ClassSecurityInfo
from Globals import InitializeClass
from Globals import DTMLFile
from Acquisition import aq_inner, aq_parent


from Products.ZCatalog.ZCatalog import ZCatalog
from Products.ZCTextIndex.ParseTree import ParseError
from Products.CMFPlone.CatalogTool import CatalogTool as PloneCatalogTool
from Products.CMFPlone.PloneBatch import Batch
from Products.PythonScripts.standard import urlencode

from zope.interface import implements

from Products.CMFCore.utils import SimpleRecord, getToolByName
from Products.ZCTextIndex.HTMLSplitter import HTMLWordSplitter
from Products.ZCTextIndex.Lexicon import CaseNormalizer
from Products.ZCTextIndex.Lexicon import Splitter
from Products.ZCTextIndex.Lexicon import StopWordRemover

from Products.CMFPlone.UnicodeSplitter import Splitter as UnicodeSplitter
from Products.CMFPlone.UnicodeSplitter import CaseNormalizer as UnicodeCaseNormalizer
from Products.ZCTextIndex.ZCTextIndex import PLexicon
from Products.ZCatalog.Lazy import Lazy, LazyMap
from random import randint
from config import CONTENT_TYPES, MATERIAL_TYPES, ACTIVITY_TYPES, TOOLS_TYPES, LANGUAGES, LANGUAGES_DICT, TYPE_NAMES, SUBJECT_AREAS_DICT, TARGET_GROUPS_DICT, SUBJECT_AREAS_INVERSE_DICT, TARGET_GROUPS_INVERSE_DICT, COMMUNITY_TYPES, SEARCHABLE_TYPES, DEFAULT_ICONS, TARGET_GROUP
import time
from math import log
from messagefactory_ import i18nme as _

class CatalogTool(PloneCatalogTool):

    meta_type = 'LeMill Catalog Tool'
    security = ClassSecurityInfo()
    toolicon = 'skins/lemill/tool.gif'

    __implements__ = PloneCatalogTool.__implements__

    # Originally CatalogTool had some simplified some expensive stuff from Plone's Catalog tool and added some
    # methods allowing us more control over indexing / unindexing
    
    # For LeMill 3.0, 

    security.declarePublic( 'enumerateIndexes' ) # Subclass can call
    def enumerateIndexes( self ):
        #   Return a list of ( index_name, type, extra ) tuples for the initial
        #   index set.
        #   Creator is deprecated and may go away, use listCreators!
        #   meta_type is deprecated and may go away, use portal_type!
        plaintext_extra = SimpleRecord( lexicon_id='plaintext_lexicon'
                                      , index_type='Okapi BM25 Rank'
                                      )
        htmltext_extra = SimpleRecord( lexicon_id='htmltext_lexicon'
                                     , index_type='Okapi BM25 Rank'
                                     )
        # Adding plone_lexicon as it should be, no nee dto change the name for our own >>> safer that way
        plone_extra = SimpleRecord( lexicon_id='plone_lexicon'
                                  , index_type='Okapi BM25 Rank'
                                  )

        return ( ('Title', 'ZCTextIndex', plone_extra)
               , ('Subject', 'KeywordIndex', None)
               , ('Description', 'ZCTextIndex', plone_extra)
               , ('Creator', 'FieldIndex', None)
               , ('listCreators', 'KeywordIndex', None)
               , ('SearchableText', 'ZCTextIndex', plone_extra)
               , ('Date', 'DateIndex', None)
               , ('Type', 'FieldIndex', None)
               , ('created', 'DateIndex', None)
               , ('effective', 'DateIndex', None)
               , ('expires', 'DateIndex', None)
               , ('modified', 'DateIndex', None)
               , ('allowedRolesAndUsers', 'KeywordIndex', None)
               , ('review_state', 'FieldIndex', None)
               , ('in_reply_to', 'FieldIndex', None)
               , ('meta_type', 'FieldIndex', None)
               , ('getId', 'FieldIndex', None)
               , ('path', 'PathIndex', None)
               , ('portal_type', 'FieldIndex', None)
               )

    security.declarePublic('enumerateLexicons')
    def enumerateLexicons(self):
        # Creating our own lexicon in a different way, as the automatic creation seems to be needing something to act as a StopWordRemover and fails miserably without it.
        lexicon = PLexicon('plone_lexicon', '', UnicodeSplitter(), UnicodeCaseNormalizer())
        self._setObject('plone_lexicon', lexicon)
        return (
                 ( 'plaintext_lexicon'
                 , Splitter()
                 , CaseNormalizer()
                 , StopWordRemover()
                 )
               , ( 'htmltext_lexicon'
                 , HTMLWordSplitter()
                 , CaseNormalizer()
                 , StopWordRemover()
                 )
               )
    #XXX END

    def catalog_object(self, object, uid, idxs=[],
                       update_metadata=1, pghandler=None):
        if object and hasattr(object, 'portal_type') and object.portal_type!='CollectionsFolder':
            ZCatalog.catalog_object(self, object, uid, idxs, update_metadata, pghandler=pghandler)

    def searchResults(self, REQUEST=None, **kw):
        """Calls ZCatalog.searchResults """
        return ZCatalog.searchResults(self, REQUEST, **kw)

    __call__ = searchResults


    def titleSearch(self, title='', sort_limit=0):
        """ search titles containing given string """
        query={'getState':('draft', 'public'), 'sort_on':'getScore', 'sort_order':'reverse'}
        if sort_limit:
            query['sort_limit']=sort_limit
        query['Title']='%s*' % title
        results=self.searchResults(query)
        return results

    def fulltextSearch(self, SearchableText='', sort_limit=0):
        """ search fulltext for a string """
        query={'getState':('draft', 'public'), 'sort_on':'getScore', 'sort_order':'reverse'}
        if sort_limit:
            query['sort_limit']=sort_limit
        query['SearchableText']='%s*' % SearchableText
        results=self.searchResults(query)
        return results

    ###### 'Adapters' start here
    # these are methods that are very specifically used by page templates to efficiently return just the results needed there  

    def buildResultsFromBatch(self, batch):
        """ This method takes a Batch of results (usually 30 or less, LazyMap) and returns a list of 
            *minimal metadata* about them. This metadata is in form of dictionary.
            
            The idea is that instantiating real metadata objects from batch results is still a costly process
            and with this we can show search/browse results without instantiating metadata.
            
            This method will also do some preprocessing for metadata, f.ex finds proper names for authors and builds links to them.
            
            This method is crafted for resource_list_macros.pt and if you aren't going to display results as they are displayed there, you probably
            shouldn't use this. 
            """
        d_list=[]
        lutool=getToolByName(self, 'lemill_usertool')
        ltool=getToolByName(self, 'lemill_tool')
        url_base=getToolByName(self, 'portal_url')()
        created=False
        edited=False
        if hasattr(self, 'REQUEST') and hasattr(self.REQUEST, 'form'):
            form=self.REQUEST.form
            if form:
                created= 'created' in form
                edited='edited' in form
        if hasattr(batch._sequence, '_seq'):
            seq=list(batch._sequence._seq[batch.start-1:batch.end])
            keys=['rid','getHasCoverImage','getNicename','portal_type','getLocation_country','getTags', 'Language','listCreators']
            if created:
                keys.append('created')
            elif edited:
                keys.append('getLatestEdit')
            md=self.fastMetadata(seq, keys)
            for item in md:
                if item[3] not in TYPE_NAMES:
                    continue
                d={'url':self.getUrlPath(item[0]),
                    'coverimage_url':item[1],
                    'title':item[2],
                    'readable_type':item[3],
                    'country':item[4],
                    'tags':item[5],
                    'language':item[6],
                    'authors':item[7]}
                if created:
                    d['created']=item[8] # These are in weird DateIndex format, difficult to map to actual dates
                elif edited:
                    d['edited']=item[8]
                d_list.append(d)
            if created or edited:
                now=time.gmtime() # mirrors DateIndex's conversion script to provide a compatible 'now'
                now= ( ( ( ( now[0] * 12 + now[1] ) * 31 + now[2] ) * 24 + now[3] ) * 60 + now[4] )
                useDateIndex=True                          
        else: # batch has already been converted to catalog Brains object
            for item in batch:                
                d={'url':item.getURL(),
                    'coverimage_url':item.getHasCoverImage,
                    'title':item.getNicename,
                    'readable_type':item.portal_type,
                    'country':item.getLocation_country,
                    'tags':item.getTags,
                    'language':item.Language,
                    'authors':item.listCreators}                  
                if created:
                    d['created']=item.created
                elif edited:
                    d['edited']=item.getLatestEdit
                d_list.append(d)
            now=time.time()
            useDateIndex=False
        tag_base='/'.join((url_base,'search?index_type=tags&q='))
        lang_base='/'.join((url_base,'search?language='))
        country_base='/'.join((url_base,'community/browse?country='))
        
        # Manipulate raw index data to something more useful
        for item in d_list:
            if callable(item['title']):
                item['title']=item['title']()
            url=item['url']
            if item['coverimage_url']:
                item['coverimage_url']='/'.join((url,'coverImage'))
            else:
                item['coverimage_url']='/'.join((url_base, DEFAULT_ICONS[item['readable_type']]))
            ll=item['language']
            if ll:
                item['language']=LANGUAGES_DICT[ll]
                item['language_link']=''.join((lang_base,ll))
            item['tags']=[(tag, ''.join((tag_base, tag))) for tag in item['tags']]
            item['authors']=[self.fastLink(author) for author in item['authors']]
            if item['country']:         
                item['country_link']=''.join((country_base,item['country']))
            item['readable_type']=TYPE_NAMES.get(item['readable_type'], None)[0]
            if 'created' in item: 
                item['timedif']=ltool.getTimeDifference(item['created'], now=now, useDateIndex=useDateIndex)
            elif 'edited' in item:
                item['timedif']=ltool.getTimeDifference(item['edited'], now=now, useDateIndex=useDateIndex)
        return d_list

    def createBrowsingOptions(self, REQUEST, results=None, **kw):
        """ This is used by browse_macros.pt and other places that need to display filters 
            based on the search results available and the section where the results are displayed.
        
            This method analyzes both request and results to decide what filter fields it should display and if it should
            preselect something or count occurences for each value. The logic for this may look hairy,
             but it can be understood by approaching this case-by-case:
            'If we are browsing in Content section, and there are >1000 results, what filters we should show'           
        """
        force_language=True
        src=dict(REQUEST.form.items())
        # section determines what kinds of options there are:
        path=REQUEST['ACTUAL_URL'].split('/')
        if 'portfolio' in kw:
            do_languages=False
            do_subject_areas=True
            do_target_groups=False
            do_types=True
            type_selection=SEARCHABLE_TYPES
            do_edited=True
            do_tags=False
            count_indexes=('portal_type','getSubject_area')
        elif 'content' in path:
            do_languages=True
            do_subject_areas=True
            do_target_groups=True
            do_types=True
            if 'drafts' in path or 'published' in path:
                type_selection=CONTENT_TYPES
            else:
                type_selection=MATERIAL_TYPES+('LeMillPrintResource',)
            do_edited=True
            do_tags=True
            count_indexes=('Language','getTarget_group','portal_type','getSubject_area')
        elif 'methods' in path or 'tools' in path:
            do_languages=True
            do_subject_areas=False
            do_target_groups=False
            do_types=False
            do_edited=True
            do_tags=True
            count_indexes=('Language',)
        elif 'community' in path:
            do_languages=True
            do_subject_areas=True
            do_target_groups=False
            do_types=True
            type_selection=COMMUNITY_TYPES
            do_edited=True
            do_tags=True
            count_indexes=('getLanguage_skills','portal_type','getSubject_area')
        else:
            do_languages=True
            force_language=False
            do_subject_areas=True
            do_target_groups=True
            do_types=True
            type_selection=SEARCHABLE_TYPES
            do_edited=True
            do_tags=False
            count_indexes=('Language','getTarget_group','portal_type','getSubject_area')
        
        d={'do_languages':do_languages, 'do_subject_areas':do_subject_areas,'do_target_groups':do_target_groups,'do_types':do_types,'do_edited':do_edited, 'do_tags':do_tags, 'q':'', 'state':'', 'author':'', 'created':''}


        # now see if the results should be counted and provide a dictionary of counted values if necessary
        if results: #and len(results)<1000:
            count_results= dict(zip(count_indexes, self.fastCount(results, count_indexes)))
            count=len(results)
        else:
            count=0

        # Languages: two lists, one for probable options and other for improbable
        if do_languages:
            lang_dict = getToolByName(self, 'lemill_tool').language_dict
            common_language_codes=getToolByName(self,'lemill_usertool').getLanguages()
            selected_language= src.get('language','')
            was_empty=not selected_language
            if was_empty:                
                if force_language:
                    selected_language=common_language_codes[0]
                else:
                    selected_language='all'
            common_languages=[]
            rare_languages=[('all','All',int(selected_language=='all'),0)]
            if count:
                if 'Language' in count_results:
                    langs=count_results['Language']
                else:
                    langs=count_results['getLanguage_skills']
                for langcode in common_language_codes:
                    common_languages.append((langcode, lang_dict[langcode], int(langcode==selected_language), langs.get(langcode, 0)))
                for langcode in langs:
                    rare_languages.append((langcode, lang_dict[langcode], int(langcode==selected_language), langs[langcode]))
            else:
                all_languages=list(LANGUAGES)[1:] # Remove 'language neutral' from options
                all_languages[0]=('','any language')
                all_language_codes=[l[0] for l in all_languages]
                for langcode in common_language_codes:
                    common_languages.append((langcode, lang_dict[langcode], int(langcode==selected_language), 0))
                    all_language_codes.remove(langcode)            
                for langcode in all_language_codes:
                    if langcode:
                        rare_languages.append((langcode, lang_dict[langcode], int(langcode==selected_language), 0))
            d['common_languages']=common_languages
            d['rare_languages']=rare_languages
            d['lang_disabled']=False # portal_type==Piece can set this to true
            if selected_language!='all':
                d['lang_filter']='language=%s&' % selected_language
            else:
                d['lang_filter']=''
        if do_subject_areas:
            selected=src.get('subject_area','')
            if selected:
                all_count='?'
            else:
                all_count=count
            subject_areas=[('','All',int(not selected), all_count)]
            if count:
                counts=count_results['getSubject_area']
                for sa_key,sa_full in sorted(SUBJECT_AREAS_DICT.items()):
                    if sa_full in counts:
                        subject_areas.append((sa_key, sa_full, int(selected==sa_key), counts[sa_full]))
            else:
                for sa_key,sa_full in sorted(SUBJECT_AREAS_DICT.items()):
                    subject_areas.append((sa_key, sa_full, int(selected==sa_key), 0))

            d['subject_area']=subject_areas
            d['subjs_disabled']=False # portal_type in ['Piece','Activity','Tool',...] will disable this
        if do_target_groups:
            selected=src.get('target_group','')
            if selected:
                all_count='?'
            else:
                all_count=count
            target_groups=[('','All',int(not selected),all_count)]
            if count:
                counts=count_results['getTarget_group']
                for tg_key,tg_full in sorted(TARGET_GROUPS_DICT.items()):
                    if tg_full in counts:
                        target_groups.append((tg_key, tg_full, int(selected==tg_key), counts[tg_full]))
            else:
                for tg_key,tg_full in sorted(TARGET_GROUPS_DICT.items()):
                    target_groups.append((tg_key, tg_full, int(selected==tg_key), 0))
            d['target_group']=target_groups
            d['targs_disabled']=False # portal_type in ['Piece','Activity','Tool',...] will disable this
        if do_types:
            selected=src.get('type','')
            if selected:
                all_count='?'
            else:
                all_count=count
            types=[('','All',int(not selected),all_count)]
            if count:
                counts=count_results['portal_type']
                for type_key in type_selection:
                    if type_key in counts:
                        types.append((type_key, TYPE_NAMES[type_key][1], int(selected==type_key), counts[type_key]))
            else:
                for type_key in type_selection:
                    types.append((type_key, TYPE_NAMES[type_key][1], int(selected==type_key), 0))
            d['type']=types
            if selected:
                if selected in ['Piece','Activity','Tool']:
                    d['targs_disabled']=True
                    d['subjs_disabled']=True
                if selected=='Piece':
                    d['lang_disabled']=True

        if do_edited:
            try:
                selected=int(src.get('edited',0))
            except ValueError: 
                selected=0
            d['edited']=[('',_('Any time'),int(not selected),0), (365,_('Last year'),int(selected==365),0), (30,_('Last month'),int(selected==30),0), (7,_('Last week'),int(selected==7),0), (1,_('Yesterday'),int(selected==1),0)]
        # Previous search term
        if 'q' in src:
            d['q']=src['q']
        if 'state' in src:
            d['state']=src['state']
        if 'author' in src:
            d['author']=src['author']            
        return d


    def decideBrowsingSubType(self):
        """ Browse result page main title needs to know what kind of browsing is going on """
        form=self.REQUEST.form        
        if 'state' in form:
            if form['state']=='public': return 'published'
            if form['state']=='draft': return 'drafts'
        path=self.REQUEST['ACTUAL_URL'].split('/')
        for key in ['content','methods','tools','community']:
            if key in path: return key
        return ''            

    def decideSearchSubType(self):
        """ Search result page main title needs to know what kind of search is going on """
        form=self.REQUEST.form        
        if 'created' in form:
            if 'type' in form and form['type']=='MemberFolder':
                return 'new_members'
            else:
                return 'new_resources'
        if 'edited' in form and 'q' not in form:
            return 'recent_edits'
        return 'search'

    def browsingSearch(self, REQUEST=None, **kw):        
        """ This is a general purpose catalog search that can convert readable keywords from request into actual search terms.
        recognized keywords: language, subject_area, target_group, type, tags, state, author, created, edited, group, country.
        
        These same keywords are used all over in interface to build selection boxes, select page headers etc. 
        The actual search indexes like 'getTags' etc. are used only here and should not be used elsewhere.                
        """
        src=REQUEST.form
        if '-C' in src:
            del src['-C']
        if not (src or kw):
            path=REQUEST['ACTUAL_URL'].split('/')
            if 'browse' in path or 'search' in path or 'cloud' in path:
                return []
        keywords={'sort_on':'getScore','sort_order':'reverse','getState':('draft', 'public')}
        keywords.update(src)
        keywords.update(kw)

        find_empty_values=keywords.get('empties','')

        if 'language' in keywords:
            if keywords['language']=='all':
                keywords['Language']=''
            else:            
                keywords['Language']=keywords['language']
            del keywords['language'] 
        if 'subject_area' in keywords:
            if keywords['subject_area'] in SUBJECT_AREAS_DICT:
                keywords['getSubject_area']=SUBJECT_AREAS_DICT[keywords['subject_area']]
            del keywords['subject_area'] 
        if 'target_group' in keywords:
            if keywords['target_group'] in TARGET_GROUPS_DICT:
                keywords['getTarget_group']=TARGET_GROUPS_DICT[keywords['target_group']]
            del keywords['target_group'] 
        if 'tags' in keywords:
            keywords['getTags']=keywords['tags']
            del keywords['tags']
        if 'author' in keywords and keywords['author']:
            keywords['listCreators']=keywords['author']
            del keywords['author']
        if 'group' in keywords and keywords['group']:
            keywords['getRawGroupEditing']=keywords['group']
            del keywords['group']
        if 'state' in keywords:
            if keywords['state']=='draft':
                lutool=getToolByName(self,'lemill_usertool')                
                if 'listCreators' in keywords and keywords['listCreators'] == lutool.getAuthenticatedId(): # only allow authenticated author to view privates
                    keywords['getState']=('draft', 'private')
                else:
                    keywords['getState']='draft' 
            elif keywords['state']=='private': # don't allow manually mungling browse terms
                return []
            else:
                keywords['getState']=keywords['state']            
            del keywords['state']
        if 'country' in keywords:
            keywords['getLocation_country']=keywords['country']
        if 'type' in keywords:
            if keywords['type']=='lr':
                keywords['portal_type']=list(CONTENT_TYPES)+['Activity','Tool']
            else:
                keywords['portal_type']=keywords['type']
            del keywords['type'] 
        else: # portal_type is determined by location 
            path=REQUEST['ACTUAL_URL'].split('/')
            if 'portfolio' in kw:
                keywords['portal_type']=list(CONTENT_TYPES)+['Activity','Tool','Collection']
                del keywords['portfolio']
            elif 'content' in path:
                if 'listCreators' in keywords: # my published, my drafts and portfolios should display pieces
                    keywords['portal_type']=list(CONTENT_TYPES)
                else:
                    keywords['portal_type']=list(MATERIAL_TYPES)+['LeMillPrintResource']
            elif 'methods' in path:
                keywords['portal_type']='Activity'
            elif 'tools' in path:
                keywords['portal_type']='Tool'
            elif 'community' in path:
                keywords['portal_type']=list(COMMUNITY_TYPES)
            else:
                keywords['portal_type']=list(SEARCHABLE_TYPES)
        if 'q' in keywords: 
            keywords['SearchableText']=keywords['q']
            del keywords['q']
        if 'created' in keywords:
            keywords['created']={'query': self.ZopeTime()-int(keywords['created']), 'range':'min'}
            keywords['sort_on']='created'
        if 'edited' in keywords:
            keywords['getLatestEdit']={'query': self.ZopeTime()-int(keywords['edited']), 'range':'min'}
            keywords['sort_on']='getLatestEdit'
            del keywords['edited']

        if not 'Language' in keywords:
            found=False
            if 'RSS' in keywords:
                found=True
            elif 'empties' in keywords:
                found=True
            elif 'base' in keywords and keywords['base']=='language':
                found=True                
            else:
                for compensating in ['SearchableText', 'listCreators', 'getTarget_group', 'getTags', 'getSubject_area', 'created', 'getLatestEdit', 'getLocation_country', 'getRawGroupEditing']:
                    if compensating in keywords:
                        found=True
                        break
            if not found:
                keywords['Language']=getToolByName(self,'lemill_usertool').getLanguages()[0]
        if 'Language' in keywords: 
            if not keywords['Language']: # Delete empty 'Language' -- now results from all languages are used
                del keywords['Language']
            elif 'portal_type' in keywords: # If browsing for MemberFolders, ignore Language, use getLanguage_skills instead
                if 'MemberFolder' in keywords['portal_type'] or 'GroupBlog' in keywords['portal_type']:
                    keywords['getLanguage_skills']=keywords['Language']
                    del keywords['Language']

        if 'portal_type' in keywords:
            if not isinstance(keywords['portal_type'], list) and keywords['portal_type']=='Piece':
                # ignore language, subject_area and target_group:
                for k in ['Language','getTarget_group','getSubject_area']:
                    if k in keywords:
                        del keywords[k] 

        for k in ['base','portfolio','RSS','empties']:
            if k in keywords:
                del keywords[k]
        try:
            results = self.searchResults(keywords)                
        except ParseError:
            results = []
        if find_empty_values:
            results = self.fastFindEmptyValues(results, find_empty_values)
        return results

    #### Fast catalog handling ###########################################

    def wakeLazy(self, lazy):
        new=[]
        for l in lazy._seq:
            if isinstance(l, Lazy):
                new.extend(self.wakeLazy(l))
            else:
                new.append(l) 
        return new

    def fastPick(self, lazy_results, top):
        """ Takes Lazy results and picks one random metadata obj from top """
        tries=4
        choice=randint(0,top)
        while tries: # since we don't know how long the list is we need to use trial and error
            # to find if the random index has corresponding item
            i=0
            for item in lazy_results._seq:
                if choice==i:
                    return lazy_results._func(item)
                i+=1
            choice/=2
            tries-=1
        return lazy_results._func(item)

    def getBaseUrl(self):
        """ Just absolute url, hopefully this gets cached ( it's fast, checked )"""
        return aq_parent( aq_inner(self) ).absolute_url()

    def getUrlPath(self,data_obj):
        """ Build a working url from catalog path data """
        path=self.getpath(data_obj).split('/')[1:]
        url_base=self.getBaseUrl().split('/')
        if url_base[-1]==path[0]:
            path=path[1:] # remove lemill-server
        return '/'.join(url_base+path)

    def fastLink(self, user):
        """ Finds an user from catalog and returns a tuple containing nicename and link 
        because this gets asked so often and it is relatively expensive to build, we cache them in non-permanent dict here in catalog tool
        """
        if not hasattr(self, 'author_cache'):
            self.author_cache={}
        if not user in self.author_cache:
            res=self.searchResults(Creator=user, portal_type='MemberFolder')
            for l in res._seq:
                authortuple= (self.getEntry('getNicename', l), self.getUrlPath(l))
                self.author_cache[user]=authortuple
                return authortuple
        else:
            return self.author_cache[user]
        return (user,'')        

    def getEntry(self, index, key, empty=[]):
        val=self._catalog.getIndex(index).getEntryForObject(key, empty)
        if callable(val):
            val=val()
        return val        
            
    def fastLinks(self, results, limit):
        """ Takes Lazy results and returns a list of tuples (title, url, portal_type) """
        def safeData(index, key):
            v=index.getEntryForObject(key, [])
            if callable(v):
                return v()
            return v

        def wakeUp(lazy, c):
            new=[]
            for l in lazy._seq:
                if isinstance(l, Lazy):
                    c,newer=wakeUp(l,c)
                    new.extend(newer)
                else:
                    new.append((safeData(title_index, l), self.getUrlPath(l), safeData(type_index, l)))
                    c+=1
                if c==limit:
                    break
            return c,new
        title_index=self._catalog.getIndex('getNicename')
        type_index=self._catalog.getIndex('portal_type')
        c,new=wakeUp(results,0)        
        return new

    def fastMetadata(self, results, indexes, cut=0):
        """ Takes Lazy results and returns a list of tuple for values from indexes.
        Indexes is a string or a tuple of strings.
        This is an order of magnitude faster than getting CatalogBrains for each object"""
        c=0
        def safeData(index, key):
            v=index.getEntryForObject(key, [])
            if callable(v):
                return v()
            return v

        def wakeUp(lazy,c):
            new=[]
            if isinstance(lazy, list):
                seq=lazy
            else:
                seq=lazy._seq
            for l in seq:
                if isinstance(l, Lazy):
                    new.extend(wakeUp(l,c))
                else:
                    values=[]    
                    for index in index_sources:
                        if index:
                            values.append(safeData(index, l))
                        else:
                            values.append(l) 
                    new.append(tuple(values))
                    c+=1
                    if c==cut:
                        break
            return new
        def wakeUpFaster(lazy, c):
            new=[]
            if isinstance(lazy, list):
                seq=lazy
            else:
                seq=lazy._seq
            for l in seq:
                if isinstance(l, Lazy):
                    new.extend(wakeUpFaster(l,c))
                else:
                    new.append(index.getEntryForObject(l, []))
                c+=1
                if c==cut:
                    break
            return new
        if isinstance(indexes, tuple) or isinstance(indexes, list):
            index_sources=[ind!='rid' and self._catalog.getIndex(ind) for ind in indexes]
            new=wakeUp(results,c)
        else:
            index=self._catalog.getIndex(indexes)
            new=wakeUpFaster(results,c)
        return new
        
    def fastCount(self, results, indexes):
        """ Takes Lazy results and returns a dictionary or tuple of dictionaries for values and their counts from indexes. 
        Index is a string or tuple of strings. 
        This is an order of magnitude faster than getting CatalogBrains for each object"""

        def safeData(index, key):
            v=index.getEntryForObject(key, [])
            if callable(v):
                return v()
            return v
        
        def wakeUpAndCount(lazy):
            for l in lazy._seq:
                if isinstance(l, Lazy):
                    wakeUpAndCount(l)
                else:
                    values=[]    
                    for dic, index in indexes:
                        val=safeData(index, l)
                        if val:
                            if isinstance(val, list):
                                for v in val:                        
                                    dic[v[:50]]=dic.get(v[:50],0)+1
                            elif val:
                                val=str(val)[:50]
                                dic[val]=dic.get(val,0)+1
        def wakeUpFasterAndCount(lazy):
            for l in lazy._seq:
                if isinstance(l, Lazy):
                    wakeUpFasterAndCount(l)
                else:
                    val=safeData(index, l)
                    if isinstance(val, list):
                        for v in val:                        
                            dic[v[:50]]=dic.get(v[:50],0)+1
                    elif val:
                        val=str(val)[:50]
                        dic[val]=dic.get(val,0)+1
        index=None
        if isinstance(indexes, tuple):
            if not results:
                return [{} for ind in indexes]
            indexes=[({}, self._catalog.getIndex(ind)) for ind in indexes]
            new=wakeUpAndCount(results)
            return tuple([dic for dic,index in indexes])
        else:
            if not results:
                return {}
            index=self._catalog.getIndex(indexes)
            dic={}
            new=wakeUpFasterAndCount(results)
            return dic


    def fastFindEmptyValues(self, results, index_to_look):
        """ Takes Lazy results and return those results where the given index returns empty"""
        new_results=[]
        def safeData(index, key):
            v=index.getEntryForObject(key, [])
            if callable(v):
                return v()
            return v
        
        def wakeUpAndFilter(lazy):
            for l in lazy._seq:
                if isinstance(l, Lazy):
                    wakeUpAndFilter(l)
                else:
                    val=safeData(index, l)
                    if not val:
                        new_results.append(l)
        if not results:
            return []
        index=self._catalog.getIndex(index_to_look)
        wakeUpAndFilter(results)
        assert isinstance(results,LazyMap)
        return LazyMap(results._func, new_results, len(new_results))


    def getSomeMetadataForRID(self, rid, md_fields):
        record = self._catalog.data[rid]
        schema = self._catalog.schema
        if isinstance(md_fields, (tuple, list)):
            result = {}
            for md in md_fields:
                result[md]= record[schema[md]]
            return result
        else:
            return record[schema[md_fields]]


    ##############################      Clouds      ########################  

    def buildCloudData(self, results, request=None, size=8, portfolio=False):
        """ Build tag cloud result tuples (name, tagsize, obj_url, tag_value, nicename) for given form from result set """ 
        def adjustTag(val, steps=8):
            # helper method to adjust hit count of this tag to relative size (1,...,8) 
            try:
                val=int((steps*log(val-mincount,2))/log(maxcount-mincount,2))
            except (OverflowError, ZeroDivisionError):
                val=0
            if not val:
                val=1
            return val
        lemill_tool = getToolByName(self, 'lemill_tool')
        if not results:
            return []
        if portfolio:
            cloud_index='getTags'
            mf=self.getMemberFolder()            
            link_root='%s/portfolio?tags=' % mf.absolute_url()
        else:        
            src=dict(request.form.items())
            if 'base' not in src:
                return [] 
            portal_url=getToolByName(self, 'portal_url')()
            path=request['ACTUAL_URL'].split('/')
            if 'content' in path: 
                link_base='/'.join((portal_url,'content','browse'))
            elif 'methods' in path:
                link_base='/'.join((portal_url,'methods','browse'))
            elif 'tools' in path:
                link_base='/'.join((portal_url,'tools','browse'))
            elif 'community' in path:
                link_base='/'.join((portal_url,'community','browse'))
            else:
                link_base='/'.join((portal_url,'browse'))        
            lang_part=''
            base=src['base']
            language=src.get('language','')
            type_restriction=src.get('type','')
            if language:
                lang_part='language=%s&' % language
            if type_restriction:
                lang_part+='type=%s&' % type_restriction
            title_cloud=False
            if base=='language':
                link_root='%s?%slanguage=' % (link_base, lang_part)
                cloud_index='Language'
                if 'type' in src:
                    if 'GroupBlog' in src['type'] or 'MemberFolder' in src['type']:
                        cloud_index='getLanguage_skills'     
            elif base=='tags':
                link_root='%s?%stags=' % (link_base, lang_part)       
                cloud_index='getTags'
            elif base=='subject_area':
                link_root='%s?%ssubject_area=' % (link_base, lang_part)       
                cloud_index='getSubject_area'
            elif base=='target_group':
                link_root='%s?%starget_group=' % (link_base, lang_part)       
                cloud_index='getTarget_group'
            elif base=='country':
                link_root='%s?%scountry=' % (link_base, lang_part)
                cloud_index='getLocation_country'
            elif base=='title':
                resultlist=self.fastMetadata(results, ('sortable_title','getScore','getNicename', 'path'), cut=100)
                if not resultlist:
                    return []
                maxcount=resultlist[0][1]
                mincount=resultlist[-1][1]             
                resultlist.sort()
                resultlist=[(x[2], adjustTag(x[1], steps=6), x[3]) for x in resultlist if x[2]] 
                return resultlist
        hits=self.fastCount(results, cloud_index)
        if not hits:
            return []
        resultlist=zip(hits.values(),hits.keys())
        resultlist.sort(reverse=True)
        maxcount=resultlist[0][0] # first!
        resultlist = resultlist[:100]
        mincount=resultlist[-1][0] # last!
        # At this point resultlist= [(number_of_hits, tag_text),...]
        # It should end up as: [(displayed_text, number_of_hits, link_url)...]
        if cloud_index=='Language' or cloud_index=='getLanguage_skills':
            resultlist=[(lemill_tool.getPrettyLanguage(x[1]), adjustTag(x[0]), ''.join((link_root, x[1]))) for x in resultlist]            
        elif cloud_index=='getSubject_area':
            resultlist=[(x[1], adjustTag(x[0]), ''.join((link_root, SUBJECT_AREAS_INVERSE_DICT.get(x[1], x[1])))) for x in resultlist]            
        elif cloud_index=='getTarget_group':
            resultlist=[(x[1], adjustTag(x[0]), ''.join((link_root, TARGET_GROUPS_INVERSE_DICT.get(x[1], x[1])))) for x in resultlist]            
        else:
            resultlist=[(x[1], adjustTag(x[0]), ''.join((link_root, x[1]))) for x in resultlist]            
        resultlist.sort()
        return resultlist


    ########### Front page top lists ############
    
    def getTopFive(self, results=None, key_index=None, link_body='', clean=False, cache_key=''):
        """ Returns top five results for key_index (getTags, getSubject_area...) for certain language """
        if clean or not hasattr(self, 'top5results'):
            self.top5results={'content':{}, 'methods':{}, 'tools':{}, 'community':{}}
        data=self.top5results
        cache_key=cache_key or key_index
        path=self.REQUEST['ACTUAL_URL'].split('/')
        for section in ['content','methods','tools','community',None]:
            if section in path:
                break
        if not section:
            return []
        ilanguage=getToolByName(self, 'portal_languages').getLanguageCookie() or 'en'
        if cache_key in data[section]:
            top5lists=data[section][cache_key]
        else:
            top5lists={}
            data[section][cache_key]=top5lists
        if ilanguage in top5lists:
            return top5lists[ilanguage]
        else:
            tops=self.fastCount(results, key_index)
            resultlist=zip(tops.values(),tops.keys())
            resultlist.sort(reverse=True)
            resultlist=resultlist[:5]
            if key_index=='getSubject_area':
                resultlist=[(x[1], ''.join((link_body, SUBJECT_AREAS_INVERSE_DICT.get(x[1], x[1])))) for x in resultlist]
            elif key_index=='getTarget_group':
                resultlist=[(x[1], ''.join((link_body, TARGET_GROUPS_INVERSE_DICT.get(x[1], x[1])))) for x in resultlist]
            else:
                resultlist=[(x[1], ''.join((link_body, x[1]))) for x in resultlist]
            top5lists[ilanguage]=resultlist
            return resultlist

    ############################# batch ##########################

    def batch(self, results=None, request=None):
        """ Use Plone's batch """
        if request and 'b_start' in request.form:
            try:
                b_start=int(request.form['b_start'])
            except ValueError:
                b_start=0
        else:
            b_start=0  
        b= Batch(results, 30, b_start, orphan=1)    
        return b  

    def batchBaseUrl(self):
        """ Keep all other parametres as they are, but add or change 'b_start' """
        form=self.REQUEST.form
        if 'b_start' in form:
            del form['b_start'] 
        last_url='?'.join((self.REQUEST.ACTUAL_URL, urlencode(form)))
        new= last_url+'&b_start='
        return new
        
CatalogTool.__doc__ = PloneCatalogTool.__doc__

InitializeClass(CatalogTool)