root/trunk/LargeSectionFolder.py

# Copyright 2006 by the LeMill Team (see AUTHORS)
#
# This file is part of LeMill.
#
# LeMill is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# LeMill is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with LeMill; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

from config import *
from Products.Archetypes.public import *
from Products.ATContentTypes.content.folder import ATFolder, ATBTreeFolderSchema, ATBTreeFolder
from Products.Archetypes.public import registerType
from Products.CMFDefault.exceptions import DiscussionNotAllowed
from Products.CMFCore.utils import getToolByName
from Products.CMFPlone.utils import transaction_note
from ZPublisher.HTTPRequest import FileUpload
from DocumentTemplate import sequence
from random import sample, shuffle
from DateTime import DateTime
import re, datetime, urllib, cgi
from AccessControl import ClassSecurityInfo
from permissions import MODIFY_CONTENT, ADD_CONTENT_PERMISSION, ModerateContent
from messagefactory_ import i18nme as _
from CommonMixIn import CommonMixIn
from Products.ZCatalog.Lazy import Lazy
from config import to_unicode
import xml.dom.minidom

import time,copy,logging,transaction

LOG = logging.getLogger('Zope.ZCatalog')

# This is a hack since I can't figure a way to get the class object of our content types before creating an instance of that class. So we have a dictionary we can use instead. 
DEFAULT_LOCATIONS = {
    'Piece':'content/pieces',
    'LeMillReference':'content/references',
    'LeMillPrintResource':'content/printresources',
    'MultimediaMaterial':'content/webpages',
    'ExerciseMaterial':'content/exercises',
    'PILOTMaterial':'content/pilots',
    'PresentationMaterial':'content/presentations',
    'LessonPlan':'content/lessonplans',
    'SchoolProjectMaterial':'content/schoolprojects',
    'Activity':'methods',
    'Tool':'tools',
    'MemberFolder':'community/people',
    'GroupBlog':'community/groups'
    }


communityschema= ATBTreeFolderSchema + Schema((
    LinesField('collaboration_proposals',
        default= [],
        widget = LinesWidget(
            visible = {'view':'invisible', 'edit':'invisible'},
            )
        )

))


class LeMillFolder:
    """ Methods common to all folderish objects """

    security = ClassSecurityInfo()

    #security.declareProtected(ADD_CONTENT_PERMISSION, 'invokeFactory')
    def invokeFactory(self, type_name, id, RESPONSE=None, *args, **kw):
        """ Checks that the path of object creation is reasonable, 
        then invokes the portal_types tool and finally sets state. 
        """
        pt = getToolByName(self, 'portal_types')
        id=pt.constructContent(type_name, self, id, RESPONSE, *args, **kw)
        if id:
            obj= getattr(self, id)
            if obj.hasComplexWorkflow():
                if obj.getHideDrafts():
                    obj.setState('private')
                else:
                    obj.setState('draft')
            else:
                try:
                    obj.setState('public')
                except:
                    pass
            # Setting permissions to newly created object
            # This should override the ones set by workflow
            obj.setPermissions()
        return id

    def getDefaultPathFor(self, portal_type, as_list=False):
        """ Returns the path that this object should have if it is in its right place (not trashed) """        
        #print 'physical: %s' % '/'.join( self.getPhysicalPath() )
        path= DEFAULT_LOCATIONS.get(portal_type, None)
        if not path:
            return None
        path='/%s/%s' % (self.getPhysicalPath()[1],path)
        #print 'default: %s' % path
        if as_list:
            return path.split('/')
        else:
            return path

    def getFolderContents(self):
        """ This shouldn't be needed """
        return []
        

class LargeSectionFolder(CommonMixIn, LeMillFolder, ATBTreeFolder):
    """Section Folder"""
        
    archetype_name = "Large Section Folder"
    meta_type = "Large Section Folder"
    security = ClassSecurityInfo()
    isAnObjectManager = 1
    global_allow= 1
    portlet=''
    subsections=[]



    ############### Not found redirector #####################

#    def __bobo_traverse__(self, REQUEST, entry_name=None):
#        """ try looking into subfolder if traverse is otherwise failing """
#        try:
#            l = BaseObject.__bobo_traverse__(self, REQUEST, entry_name)
#            return l
#        except AttributeError, entry_name:
#            print 'looking for %s' % entry_name
#            raise AttributeError(entry_name)        
#
#
#
#    def __aabobo_traverse__(self, REQUEST, entry_name=None):
#        """ try looking into subfolder if traverse is otherwise failing """
#        try:
#            l = BaseObject.__bobo_traverse__(self, REQUEST, entry_name)
#        except AttributeError:
#            print 'looking for %s' % entry_name 
#            l = self.lookInSubsections(entry_name)
#            if l:
#                l.REQUEST.RESPONSE.redirect(l.absolute_url(), status=301)
#            else:
#                raise AttributeError(entry_name)
#        #if hasattr(l, 'im_func'):
#        #    return l()
#        return l

    
    def lookInSubsections(self, obj_id):
        """ Check if subsection has this object and redirect if does """
        for sub in self.__class__.subsections:
            if sub!=self.getId():
                sub=getattr(self, sub)
                if sub.has_key(obj_id):
                    return getattr(sub, obj_id)
        raise AttributeError(obj_id)


    ############### Misc. useful getters #####################


    def fixMissingCatalogObjects(self, portal_type, start=0, end=10000):
        """ check if cataloged uid matches with object uid """
        pc = getToolByName(self, 'portal_catalog')
        results = pc({'portal_type':portal_type})
        start=int(start)
        end=int(end)
        c=0
        f=0
        for r in list(results):
            c=c+1
            if c<start:
                continue
            if c>end:
                break
            o=None
            try:
                o= r.getObject()
            except:
                f=f+1
                print '%s failed to get object %s' % (str(c), r.getPath())
                pc.uncatalog_object(r.getPath())
            if o and o.UID()!=r.UID:
                print "%s UID mismatch: object %s's uid is %s but metadata is %s" % (str(c), r.getId, o.UID(), r.UID)
            else:
                print ".",
        print '%s failures out of %s catalog objects' % (str(f), str(c))

    def getAllContentTypes(self):
        """ returns a list of all content types """
        return ALL_CONTENT_TYPES

    def getMaterialTypes(self, no_references=False):
        """ returns a list of material types """
        if no_references:
            return [x for x in MATERIAL_TYPES if x!='LeMillReference']
        else:
            return MATERIAL_TYPES

    def getContentTypes(self):
        """ returns a list of material types """
        return CONTENT_TYPES

    def getActivityTypes(self):
        """ returns a list of activity types """
        return ACTIVITY_TYPES

    def getToolsTypes(self):
        """ returns a list of activity types """
        return TOOLS_TYPES

    def getFeaturedTypes(self):
        """ returns a list of featured types """
        return MATERIAL_TYPES + ACTIVITY_TYPES + TOOLS_TYPES

    def canDeleteOnCancel(self):
        """ Should be called on resource contexts, so NO """
        return False 


    def prettyMethodNameDictionary(self):
        """ Helper method for finding displayable names for search fields """
        # better to do this in config..
        return FIELD_METHODS_DISPLAY_STRINGS                     


    def getNameOfResourceType(self,restype, lower=True):
        """Get human-readable name of resource - first try TYPE_NAMES, then TEMPLATES"""
        name=TYPE_NAMES.get(restype, restype)
        if isinstance(name, tuple):
            name=name[0]        
        if lower:
            return name.lower()
        else:
            return name

    def getSectionFolder(self, bottom=False):
        """ return this or if bottom, then tries to look if there is a section folders beyond this """
        if not bottom:
            return self
        if isinstance(self.aq_parent,LargeSectionFolder):
            return self.aq_parent.getSectionFolder(bottom=True)
        else:
            return self

    def amIManager(self):
        """Check whether I'm a manager."""
        lutool=getToolByName(self, 'lemill_usertool')        
        roles = lutool.getAuthenticatedMember().getRolesInContext(self)
        return 'Manager' in roles 

    def canIModerate(self):
        lutool=getToolByName(self, 'lemill_usertool')        
        roles = lutool.getAuthenticatedMember().getRolesInContext(self)
        return 'Manager' in roles or 'Reviewer' in roles

    def url_quote(self,word):
        return urllib.quote(word)


    def getDefaultIcon(self, meta_type, obj=None):
        """ general method for getting proper icon for object, used when only catalog-metadata is available """        
        address=DEFAULT_ICONS[meta_type]
        if address!='piece':
            return address
        else:
            try:
                # getObject verified
                obj=obj.getObject()
                return obj.getDefaultIcon()
            except (AttributeError, KeyError):
                return DEFAULT_ICONS['Piece']


    def getPortletPath(self):
        """ Return a path to portlet, defined in object's class """
        return self.__class__.portlet


    def getFolderContents(self):
        """ get contents by using catalog, faster than actual listing methods """
        catalog = getToolByName(self,'portal_catalog')
        results = catalog.searchResults(path="/".join(self.getPhysicalPath()), sort_on='modified', sort_order='reverse', getState="public")
        return results


    ################# Object creation ####################


    # Fixes unicode ids causing problems with copypaste
    def _checkId(self, id, allow_dup=0):
        id=str(id)
        ATBTreeFolder._checkId(self, id, allow_dup)
        

    # Override initializeArchetype to turn on syndication by default
    security.declarePrivate('initializeArchetype')
    def initializeArchetype(self, **kwargs):
        ret_val = ATBTreeFolder.initializeArchetype(self, **kwargs)
        # Enable topic syndication by default
        syn_tool = getToolByName(self, 'portal_syndication', None)
        if syn_tool is not None:
            if syn_tool.isSiteSyndicationAllowed():
                try:
                    syn_tool.enableSyndication(self)
                except: # might get 'Syndication Information Exists'
                    pass
        return ret_val        

    security.declareProtected(ADD_CONTENT_PERMISSION,'createPiece')
    def createPiece(self, container, id=None, file=None):
        new=self.lemill_invokeFactory(container, 'Piece', id, do_create=True)
        new.edit(file=file.read(), language='')
        return new

    security.declareProtected(ADD_CONTENT_PERMISSION,'lemill_invokeFactory')
    def lemill_invokeFactory(self, container, meta_type, id=None, do_create=False):
        """ add new object, edit it's title and invoke _renameAfterCreation """
        ft=getToolByName(self, 'portal_factory')
        if id is None:
            id = self.generateUniqueId(meta_type)
        path_as_list=self.getDefaultPathFor(meta_type, as_list=True)
        if path_as_list and path_as_list != list(container.getPhysicalPath()):
            container=self.restrictedTraverse(self.getDefaultPathFor(meta_type))        

        if ft.getFactoryTypes().has_key(meta_type):
            o = container.restrictedTraverse('portal_factory/' + meta_type + '/' + id)
            message = None
            transaction_note('Initiated creation of %s with id %s in %s' % (o.getTypeInfo().getId(), id, container.absolute_url()))
        else:
            new_id = container.invokeFactory(id=id, type_name=meta_type)
            if new_id is None or new_id == '':
               new_id = id
            o=getattr(container, new_id, None)
            tname = o.getTypeInfo().Title()
            message = _(u'text_message_object_created', u'${tname} has been created.', mapping={u'tname' : tname})
            transaction_note('Created %s with id %s in %s' % (o.getTypeInfo().getId(), new_id, container.absolute_url()))
        if do_create: # finalizes object
            o=ft.doCreate(o)
        #print 'Created %s' % o
        return o


    def checkTitle(self, obj=None ,title='', objtype=''):
        """ check if title is not used anywhere in not(deleted, redirector) object, return false if it is """
        lt=getToolByName(self, 'lemill_tool')
        return lt.checkTitle(self,obj=obj, title=title, objtype=objtype)




    ##############         Resource conversion      ######################

    security.declareProtected(ModerateContent, 'convertResource')
    def convertResource(self, REQUEST):
        """ Copy values of some fields to new object and redirect old references """
        allowed_types={'LeMillReference':['content','references'],'Tool':'tools','Activity':'methods','MultimediaMaterial':['content','webpages'], 'ExerciseMaterial':['content','exercises']}
        lt=getToolByName(self, 'lemill_tool')

        # Check if we have everything we need and raise error if not
        from_obj_id=REQUEST.get('from_obj_id', '')
        to_obj_type=REQUEST.get('to_obj_type', '')
        trial_run=REQUEST.get('trial_run', True)

        if trial_run:
            if trial_run=='False' or trial_run=='0':
                trial_run=False

        from_obj=getattr(self, from_obj_id,None)
        if not from_obj:
            raise "base object for resource conversion doesn't exist"

        from_obj_type=from_obj.portal_type
        if from_obj_type not in allowed_types.keys():
            raise "cannot convert from type %s" % from_obj_type
        if to_obj_type not in allowed_types.keys():
            raise "cannot transfer to type %s" % to_obj_type

        # Create to_obj
        if to_obj_type in ['Tool','Activity']:
            to_folder = getattr(self.aq_parent, allowed_types[to_obj_type])
        else:
            section = getattr(self.aq_parent, allowed_types[to_obj_type][0])
            to_folder = getattr(section, allowed_types[to_obj_type][1])
        to_obj = self.lemill_invokeFactory(to_folder, to_obj_type, do_create=True)

        from_schema=from_obj.schema
        to_schema=to_obj.schema
         
        import copy
        ignored_fields=['translation', 'translation_of','id','groups']

        # trial run, report fields that have content but cannot be converted
        if trial_run:
            problems=False
            for k in from_schema.keys():
                from_accessor = from_schema[k].getEditAccessor(from_obj)
                if k not in ignored_fields and  k not in to_schema.keys() and from_accessor and from_accessor():
                    #stupid exceptions
                    if k=='address' and from_accessor()=='http://':
                        continue
                    if k=='address' and 'location' in to_schema.keys():
                        continue
                    if k=='location' and 'address' in to_schema.keys():
                        continue
                    if k=='video' and len(from_accessor())<5:
                        continue
                    problems=True
                    lt.addPortalMessage(_('text_message_cannot_convert'), default='Cannot convert ${obj}.', mapping={'obj':k})
            to_folder._delObject(to_obj.id)
            if not problems:
                lt.addPortalMessage(_('text_message_conversion_will_be_fine'), default='Checked conversion and it will be fine. Click \'Convert resource\' to do the conversion.')
            if problems:
                lt.addPortalMessage(_("text_message_conversion_will_have_problems"), default="Indicated fields have text that cannot be copied to other resource type. Click 'Convert resource' if you want to do the conversion anyway.")
            return REQUEST.RESPONSE.redirect('%s/manage_convert?trial_run=False&from_obj_id=%s&to_obj_type=%s' % (from_obj.absolute_url(), from_obj.id, to_obj_type))


        # Helper method for copying fields
        def copyvalue(from_accessor, to_mutator, k):
            if not from_accessor:
                return
            val = from_accessor()
            if k=='bodyText':
                if from_obj_type == 'ExerciseMaterial':
                    if to_obj_type == 'MultimediaMaterial':
                        new_val=[]
                        for (chapter, c_type) in val:
                            if c_type in ['choice','multiple_choices','fill_in_the_blanks','open_ended']:
                                new_val.append((chapter, 'text_block'))
                            else:
                                new_val.append((chapter, c_type))
                        val=new_val
                    else:
                        val=from_obj.getOnlyRawText()
                elif from_obj_type == 'MultimediaMaterial':
                    if to_obj_type == 'ExerciseMaterial':
                        if val[0][0]=='text_block':
                            val[0][0]='guidelines'
                    else:
                        val=from_obj.getOnlyRawText()
            copied_val = None
            try:
                copied_val = copy.copy(val)
            except TypeError:
                copied_val = copy.copy(val.aq_base)
            to_mutator(copied_val)

        # Copy fields with same name
        for k in from_schema.keys():
            # list all fields here that shouldn't be copyied to new object
            if k not in to_schema.keys() or k in ignored_fields:
                continue
            from_accessor = from_schema[k].getEditAccessor(from_obj)
            to_mutator = to_schema[k].getMutator(to_obj)
            copyvalue(from_accessor, to_mutator, k)

        # Check convertable pairs of exceptions:
        for (k,l) in [('address','location')]:
            if k in from_schema.keys() and l in to_schema.keys():
                from_accessor = from_schema[k].getEditAccessor(from_obj)
                to_mutator = to_schema[l].getMutator(to_obj)
                copyvalue(from_accessor, to_mutator, k)
            elif l in from_schema.keys() and k in to_schema.keys():
                from_accessor = from_schema[l].getEditAccessor(from_obj)
                to_mutator = to_schema[k].getMutator(to_obj)
                copyvalue(from_accessor, to_mutator, k)

        # LeMillReferences use descriptions instead of bodytexts while they still have bodytexts:
        if to_obj_type=='LeMillReference':
            k='bodyText'
            from_accessor=from_schema['bodyText'].getEditAccessor(from_obj)
            to_mutator = to_schema['description'].getMutator(to_obj)
            copyvalue(from_accessor, to_mutator, k)
        if from_obj_type=='LeMillReference':
            k='description'
            from_accessor=from_schema['description'].getEditAccessor(from_obj)
            to_mutator = to_schema['bodyText'].getMutator(to_obj)
            copyvalue(from_accessor, to_mutator, k)


        # Look for references to this object
        rc=getToolByName(self, 'reference_catalog')
        uc=getToolByName(self, 'uid_catalog')
        results=rc({'targetUID':from_obj.UID()})

        # Get object with certain reference and fix it
        for refobj in results:
            res=uc({'UID':refobj.sourceUID})
            if res:
                # getObject verified
                obj=res[0].getObject()
                if refobj.relationship in ['relatesToContent', 'relatesToMethods', 'relatesToTools']:
                    obj.delete(from_obj)
                if obj.portal_type=='Collection':
                    obj.add(to_obj)

        # Copy references to pieces
        if from_obj_type in ['MultimediaMaterial','ExerciseMaterial'] and to_obj_type in ['MultimediaMaterial','ExerciseMaterial']:
            results=rc({'sourceUID':from_obj.UID()})
            for refobj in results:
                rc.addReference(to_obj, refobj.targetUID, 'uses')        

        # Copy history
        from_history=from_obj.getHistory()
        to_obj.setHistory(copy.copy(from_history))

        # Run post_edits
        to_obj.post_edit_rename()
        to_obj.recalculateAuthors()
        to_obj.recalculateScore()
        to_obj.reindexObject()

        # Delete old object
        self._delObject(from_obj_id)

        # Return new object
        lt.addPortalMessage(_('Resource converted succesfully.'))        
        if to_obj.portal_type in MATERIAL_TYPES:
            return REQUEST.RESPONSE.redirect('%s/view' % to_obj.absolute_url())            
        else:
            return REQUEST.RESPONSE.redirect(to_obj.absolute_url())            

        

    def wakeLazy(self, lazy):
        new=[]
        for l in lazy._seq:
            if isinstance(l, Lazy):
                new.extend(self.wakeLazy(l))
            else:
                new.append(l) 
        return new




    ################      Featured items      ###########################

    def _getSamples(self, content_types):
        lutool=getToolByName(self, 'lemill_usertool')
        lt=getToolByName(self,'lemill_tool')
        pc=getToolByName(self,'portal_catalog')
        languages=lutool.getLanguages()
        good_results=[]
        while languages:
            lang=languages.pop(0)
            results=pc(portal_type=content_types, Language=lang, getState='public', getHasCoverImage=True, sort_on='getScore', sort_order='descending')
            results=results[:30]
            shuffle(results)
            if len(results)>3:
                return results[:4]
            else:
                good_results+=results
            if len(good_results)>3:
                return good_results[:4]
        return good_results

    security.declareProtected(MANAGE_PORTAL, 'rebuildTop3s')
    def rebuildTop3s(self):
        """ set values for sections' top 3 tags, languages etc. (do this daily, or weekly: very expensive) """
        t=time.time()
        section_toplists={
            'content':['top3languages','top3subject_areas','top3target_groups','top3tags'],
            'methods':['top3languages','top3tags'],
            'tools':['top3languages','top3tags'],
            'community':['top3language_skills','top3locations','top3skills','top3interests','top3subject_areas']                
        }
        
        type_mapping = {'content':CONTENT_TYPES, 'methods':'Activity','tools':'Tool','community':'MemberFolder'} # Groups are handled separately         
        category_mapping={'top3languages':'Language', 'top3subject_areas':'getSubject_area', 'top3target_groups':'getTarget_group',  'top3tags':'getTags', 'top3language_skills':'getLanguage_skills', 'top3locations':'getLocation_country', 'top3skills':'getSkills', 'top3interests':'getInterests', 'top3group_languages':'getLanguage_skills', 'top3group_tags':'getTags','top3group_subject_areas':'getSubject_area'}
        pc=getToolByName(self, 'portal_catalog')
        # Others
        for section_name, toplist in section_toplists.items():
            results=pc(portal_type=type_mapping[section_name], getState='public', getHasCoverImage=True)
            results=self.wakeLazy(results)
            section=getattr(self.aq_parent, section_name)
            props={}
            for top3_id in toplist:
                props[top3_id]=self._getTopResults(results, category_mapping[top3_id])
            section.manage_editProperties(props)
        # Groups
        results=pc(portal_type='GroupBlog', getState='public')
        results=self.wakeLazy(results)
        for top3_id in ['top3group_languages','top3group_tags','top3group_subject_areas']:
            props[top3_id]=self._getTopResults(results, category_mapping[top3_id])
        self.aq_parent.community.manage_editProperties(props)
        return str(time.time()-t)       

        
    def _getTopResults(self, results, index_type):
        pc = getToolByName(self, 'portal_catalog')
        if index_type not in pc.indexes() or index_type not in pc.schema():
            return []
        res={}
        index=pc._catalog.getIndex(index_type)
        r=[]
        for value in results:
            value=index.getEntryForObject(value, "")
            if value and (isinstance(value, list) or isinstance(value, tuple)):
                for v in value:
                    res[v]=res.get(v,0)+1
            elif value:
                res[value]=res.get(value,0)+1
        topthree= [(n,key) for key, n in res.items() if n>1 and key]        
        topthree.sort(reverse=True)
        topthree=[key for n,key in topthree[:5]]
        return topthree


    def js_queryForPieces(self, keyword='', search_type='image', user='', collection=False):
        """ javascript is querying for pieces that are images """
        result = []
        q = {'portal_type':'Piece', 'getState': 'public'}
        if not (keyword or user):
            return [] 
        if keyword:
            q['SearchableText']=keyword
        if user:
            lutool=getToolByName(self, 'lemill_usertool')
            q['listCreators']=lutool.getAuthenticatedId()
            q['sort_on']='modified'
            q['sort_order']='descending'
        if search_type=='image' or search_type=='cover_image':
            q['getPiece_type']='image'
        elif search_type=='audio':
            q['getPiece_type']='audio'

        ltool = getToolByName(self, 'lemill_tool')
        if collection and user:
            query={'portal_type':'Piece', 'getState':'public', 'Creator':q['listCreators']}
            if 'getPiece_type' in q:
                query['getPiece_type']=q['getPiece_type']
            reftool = getToolByName(self, 'reference_catalog')
            catalog = getToolByName(self, 'portal_catalog')
            q_results=[]
            cols=ltool.searchResultsWrapper(portal_type='Collection', getState='public', Creator=q['listCreators'])
            for col in cols:
                refs=reftool(sourceUID=col.UID, relationship='relatesToContent')
                resource_uids=[x.targetUID for x in refs]
                query['UID']=resource_uids
                res=catalog(query)
                if res:
                   q_results+=res
        else:
            q_results = ltool.searchResultsWrapper(q)
        for r in q_results:
            if not r.getPiece_type:
                continue
            if r.getHasCoverImage:
                cover_url='%s/coverImage' % r.getURL()
            else:
                if r.getPiece_type=='audio':
                    cover_url='images/default_soundclip.png'
                elif r.getPiece_type=='video':
                    cover_url='images/default_movieclip.png'
                else:
                    cover_url='images/default_movieclip.png'
            image_url=''
            piece_type=search_type
            if search_type!='cover_image':
                piece_type=r.getPiece_type
                if piece_type=='image':
                    image_url='%s/image_large' % r.getURL()
            # r.Title is utf_8-encoded str, so it is changed back to unicode 
            pseudometadata = [r.UID, cover_url, r.Title.decode('utf_8'), piece_type, image_url] 
            result.append(pseudometadata)
        # javascript can handle unicode, but not utf_8-encoded stuff. Unicode shouldn't be preceded with u.
        result=str(result) # when list is turned to string, unicode strings inside are left as [..., u'titleishere', ]
        result=result.replace(" u'"," '")
        result=result.replace(' u"',' "') # and now unicode markers "u'" are removed 
        return result  

    def js_getPieceMetadata(self, uid):
        """ Get piece data so that javascript can go and inject it to a live edit form """
        def cl(s):
            return cgi.escape(to_unicode(s))
        lt=getToolByName(self,'lemill_tool')
        piece=lt.getObjectByUID(uid)
        doc=u"""<?xml version="1.0" encoding="utf-8"?>
        <piece>
            <title>%s</title>
            <description>%s</description>
            <rights>%s</rights>
            <original_author>%s</original_author>
            <tags>%s</tags>
        </piece>
        """ % (cl(piece.Title()), cl(piece.getDescription()), cl(piece.getRights()), cl(piece.getOriginalAuthor()), u', '.join([cl(tag) for tag in piece.getTags()]))
        # if there are xml-problems, it's better to have them here than in javascript
        dom = xml.dom.minidom.parseString(doc.encode('utf-8'))       
        response = self.REQUEST.RESPONSE
        response.setHeader('Content-Type', "text/xml")
        return dom.toxml()         


    def getLanguageFilter(self):
        """ returns 'language=xx&' or '' if there is a language cookie set """
        ilanguage=getToolByName(self, 'portal_languages').getLanguageCookie()
        if ilanguage:
            return 'language=%s&' % ilanguage 
        else:
            return ''

##################### Subclasses        ################################


class LargeContentFolder(LargeSectionFolder):

    archetype_name = "Large Content Folder"
    meta_type = "Large Content Folder" 

    allowed_content_types = CONTENT_TYPES +('Topic','Redirector')
    default_view = ('lemill_content_view')
    portlet='here/portlet_add_content/macros/portlet'
    filter_content_types = True
    subsections=['webpages','presentations','pilots','exercises','pieces','lessonplans','schoolprojects','printresources']
    security = ClassSecurityInfo()


    def getSamples(self):
        """ Returns content types """
        return self._getSamples(MATERIAL_TYPES)

    def getTemplate(self, template_id):
        return TEMPLATES.get(template_id, None)

    security.declareProtected(ADD_CONTENT_PERMISSION,'lemidlet_post')
    def lemidlet_post(self, REQUEST):
        """ LeMidlet will post image here...."""
        #print REQUEST.file.read()
        file = REQUEST.get('file')
        if file==None:
            return 'file not found'       
        description = REQUEST.get('description')
        type = 'Piece'
        new = self.lemill_invokeFactory(self, type, id=type+str(id(self)), do_create=True)
        new.edit(description=description, file=file.read(),language='',title=REQUEST.get('title'),tags=REQUEST.get('tags'))
        return 0

        
class LargeActivityFolder(LargeSectionFolder):

    archetype_name = "Large Activity Folder"
    meta_type = "Large Activity Folder"

    allowed_content_types = ('Activity','KB', 'Topic','Redirector')
    default_view = ('lemill_activities_view')
    portlet='here/portlet_add_activity/macros/portlet'
    subsections=[]
    filter_content_types = True

    def getSamples(self):
        """ Returns 3 methods/activities """
        return self._getSamples('Activity')
    
class LargeToolFolder(LargeSectionFolder):

    archetype_name = "Large Tool Folder"
    meta_type = "Large Tool Folder"

    allowed_content_types = ('Tool', 'Topic','Redirector')
    default_view = ('lemill_tools_view')
    portlet='here/portlet_add_tool/macros/portlet'
    subsections=[]
    filter_content_types = True

    def getSamples(self):
        """ Returns 3 tools """
        return self._getSamples('Tool')

class LargeCommunityFolder(LargeSectionFolder):

    archetype_name = "Large Community Folder"
    meta_type = "Large Community Folder"

    allowed_content_types = ('Topic','Redirector','MemberFolder','GroupBlog','ATBTreeFolder','Large Plone Folder')
    default_view = ('lemill_community_view')
    portlet='here/portlet_add_community/macros/portlet'

    filter_content_types = False
    security = ClassSecurityInfo()
    subsections=['people','groups']
    schema=communityschema


    ### If MemberFolder for logged in user does not exist, it is created here
    
    def my_page(self):
        """ Checks if user has MemberFolder and creates one if not. Returns the folder url."""        
        lutool = getToolByName(self, "lemill_usertool")
        mf_url = lutool.getMemberFolderURL()
        #print 'my_page called'
        if not mf_url:
            lutool.createMemberFolder()            
            mf_url = lutool.getMemberFolderURL()
        return mf_url 


    def getMemberFolderById(self,id):
        """ sometimes it is easier to just get it directly from btreefolder """
        if self.id=='people':
            return getattr(self, id, None)
        elif hasattr(self, 'people'):
            people=self.people
            return getattr(people, id, None)
        else:
            return getattr(self, id, None)

    def getGroupById(self,id):
        """ sometimes it is easier to just get it directly from btreefolder """
        if self.id=='groups':            
            return getattr(self, id, None)
        elif hasattr(self, 'groups'):
            groups=self.groups
            return getattr(groups, id, None)
        else:
            return getattr(self, id, None)

    def getSamples(self):
        """ Get 4 random people """  
        lutool=getToolByName(self, 'lemill_usertool')
        lt=getToolByName(self,'lemill_tool')
        pc=getToolByName(self,'portal_catalog')
        languages=lutool.getLanguages()
        good_results=[]
        while languages:
            lang=languages.pop(0)
            results=pc(portal_type='MemberFolder', getLanguage_skills=lang, getState='public', getHasCoverImage=True, sort_on='getScore', sort_order='descending')
            results=results[:30]
            shuffle(results)
            if len(results)>3:
                return results[:4]
            else:
                good_results+=results
            if len(good_results)>3:
                return good_results[:4]
        return good_results
        

    def mergeLatestThreadsInMyGroups(self):
        """ Method to help displaying latest threads (BlogPost) in section front page """
        lutool = getToolByName(self, "lemill_usertool")
        memberfolder = lutool.getMemberFolder()
        if not memberfolder:
            return []
        glist=[g.getId for g in memberfolder.getGroups(objects=False)]
        pc=getToolByName(self, 'portal_catalog')
        posts=pc({'getParentBlog':glist, 'getState':'public', 'sort_on':'created', 'sort_order':'descending','meta_type':'BlogPost'}) 
        return posts[:20]
                                    
    def getCollections(self, obj_uid=None):
        """ Show collections where object is used."""
        if not obj_uid:
            return []
        res = []
        q = { 'targetUID': obj_uid }
        qres = self.reference_catalog(q)
        for q in qres:
            v = self.reference_catalog.lookupObject(q.UID)
            if v:
                source = v.getSourceObject()
                if source.meta_type == 'Collection':
                    res.append(source)
        return res

    def getSelectablesForPortfolio(self, results):
        """Try to find which types are found in results and show them as selectables """
        pc=getToolByName(self, 'portal_catalog')
        lt=getToolByName(self, 'lemill_tool')
        types, tags = pc.fastCount(results, ('portal_type','getTags'))
        # Convert them to tuples where (value, display value)
        types=zip(map(lambda x: lt.getTypeName(x), types.keys()), types.keys(), types.values())
        tagslist=zip(tags.keys(),tags.keys(),tags.values())
        tagslist.sort()
        return {'type':types,'tag':tagslist}

    security.declareProtected(ADD_CONTENT_PERMISSION,'addGroup')
    def addGroup(self, title, join=True):
        """ Create a new group. """
        if self.getId()!='groups':
            return self.community.groups.addGroup(title, join)
        lt = getToolByName(self, 'lemill_tool')
        group_id = lt.normalizeString(title)
        group_id=self.invokeFactory('GroupBlog',group_id)
        group=getattr(self, group_id, None)
        group.edit(title=title, description="")
        if join:
            group.join_group()
        # Group object in uid_catalog gets title
        group.helper_updateCatalogUID()
        # Need to manually set _at_creation_flag to False, as processForm does not get called
        group.unmarkCreationFlag()
        return group            


class LargeTrashFolder(LargeSectionFolder):

    archetype_name = "Large Trash Folder"
    meta_type = "Large Trash Folder"

    allowed_content_types = ALL_CONTENT_TYPES
    filter_content_types = False
    default_view = ('lemill_trash_view')
    security = ClassSecurityInfo()

        
registerType(LargeContentFolder, PROJECTNAME)
registerType(LargeActivityFolder, PROJECTNAME)
registerType(LargeToolFolder, PROJECTNAME)
registerType(LargeCommunityFolder, PROJECTNAME)
registerType(LargeTrashFolder, PROJECTNAME)