ms-psb2.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
"""


Use global -simulate option for test purposes. No changes to live wiki
will be done.

Call:
    time /home/masti/venvwrapper python3 pwb.py masti/ms-psb.py -page:'Wikipedysta:PuchaczTrado/PSB/1' -maxlines:10000 -summary:'Bot uaktualnia tabelę #1'

The following parameters are supported:

&params;

-always           If used, the bot won't ask if it should file the message
                  onto user talk page.   
-outpage          Results page; otherwise "Wikipedysta:mastiBot/test" is used
-maxlines         Max number of entries before new subpage is created; default 1000
-text:            Use this text to be added; otherwise 'Test' is used
-replace:         Dont add text but replace it
-top              Place additional text on top of the page
-summary:         Set the action summary message for the edit.
"""
#
# (C) Pywikibot team, 2006-2016
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, unicode_literals

__version__ = '$Id: c1795dd2fb2de670c0b4bddb289ea9d13b1e9b3f $'
#
#
# (C) Pywikibot team, 2006-2021
#
# Distributed under the terms of the MIT license.
#
import pywikibot

from pywikibot.backports import Tuple
from pywikibot import pagegenerators

from pywikibot.bot import (
    SingleSiteBot, ConfigParserBot, ExistingPageBot, NoRedirectPageBot,
    AutomaticTWSummaryBot)


# This is required for the text that is shown when you run this script
# with the parameter -help.
docuReplacements = {'&params;': pagegenerators.parameterHelp}  # noqa: N816


import re
import datetime

actionCounters = {
    'total': 0,
    'blue': 0,
    'red': 0,
    'redirs': 0,
    'disambigs': 0
}

class Person(object):
    # Class to describe person from list
    def __init__(self):
        self.title = None
        self.dob = ''  # Date of birth
        self.dod = ''  # Date of death
        self.dobprecision = 11  # date precision for birth
        self.dodprecision = 11  # date precision for death
        self.occupation = []  # list of occupations
        self.instanceof = None
        self.wditem = ''
        self.disambig = False
        self.description = ''
        self.sex = None
        self.wdexists = False
        self.link = None
        self.comment = ''

    def personPrint(self):
        pywikibot.output('Link:%s' % self.link)
        pywikibot.output('Title:%s' % self.title)
        pywikibot.output('Wikidata exists:%s' % self.wdexists)
        if self.wdexists:
            pywikibot.output('Wikidata item:%s' % self.wditem)
            pywikibot.output('Is disambig:%s' % self.disambig)
            pywikibot.output('Is instance of:%s' % self.instanceof)
            pywikibot.output('Date of birth:%s' % self.dob)
            pywikibot.output('Date of birth precision:%s' % self.precision(self.dobprecision))
            pywikibot.output('Date of death:%s' % self.dod)
            pywikibot.output('Date of death precision:%s' % self.precision(self.dodprecision))
            pywikibot.output('Sex:%s' % self.sex)
            try:
                pywikibot.output('Occupation:%s' % ",".join(self.occupation))
            except:
                pywikibot.output('Occupation: ERROR')
        pywikibot.output('Description:%s' % self.description)
        pywikibot.output('Comment:%s' % self.comment)

    def WDexists(self):
        return(self.wdexists)

    def isDisambig(self):
        return(self.disambig)

    def whatIs(self):
        if self.sex:
            return(self.sex)
        elif self.disambig:
            return('<strona ujednoznaczniająca>')
        elif self.instanceof:
            return(self.instanceof)
        else:
            return('<brak danych>')

    def DoB(self):
        return(self.dob)

    def DoD(self):
        return(self.dod)

    def Occupation(self):
        return(", ".join(self.occupation))

    def centuryDecade(self,date): 
        #return decade of century i.e. 'lata 20. 20 wieku'
        if date > 0:
            return(pywikibot.date.formats['DecadeAD']['pl'](date))
        else:
            return(pywikibot.date.formats['DecadeBC']['pl'](date))

    def centuryFromDate(self,date):
        #return century from give wbTime
        year = date.year
        if year % 100:
            return(int(1+year/100))
        else:
            return(int(year/100))

    def millenniumFromDate(self,date):
        #return century from give wbTime
        year = date.year
        if year % 1000:
            return(int(1+year/1000))
        else:
            return(int(year/1000))

    def millennium(self,date):
        #return millennium i.e. '20 wieku'
        mill = self.millenniumFromDate(date)
        if mill > 0:
            return('%i tysiąclecie' % mill)
        else:
            return('%i tysiąclecie p.n.e.' % -1*mill)

    def century(self,date):
        #return century i.e. '20 wieku'
        if date > 0:
            return(pywikibot.date.formats['CenturyAD']['pl'](self.centuryFromDate(date)))
        else:
            return(pywikibot.date.formats['CenturyBC']['pl'](self.centuryFromDate(date)))

    def formatDate(self,date,maxPrecision=None):
        #return date formatted according to date.precision
        #pywikibot.output('Format Date:%i' % date.precision)
        if maxPrecision:
            prec = min(maxPrecision,date.precision)
        else:
            prec = date.precision
        if prec == 11: #days
            return('%d-%02d-%02d' % (date.year,date.month,date.day))
        elif prec == 10: #months
            return('%d-%02d' % (date.year,date.month))
        elif prec == 9: #years
            return('{{L|%d}}' % date.year)
        elif prec == 8: #10 years
            return(self.centuryDecade(date))
        elif prec == 7: #100 years
            return(self.century(date))
        elif prec == 6: #1000 years
            return(self.millennium(date))
        return(date)

    def setDoB(self,date):
        try:
            self.dob = self.formatDate(date,maxPrecision=9)
            self.dobprecision = date.precision
        except:
            self.dob = ''

    def setDoD(self,date):
        try:
            self.dod = self.formatDate(date,maxPrecision=9)
            self.dodprecision = date.precision
        except:
            self.dod = ''
        

    def precision(self,prec):
        #return precision text from numeric value prec
        datePrecision = {
		0:'miliard lat',
		1:'100 milionów lat',
		2:'10 milionów lat',
		3:'milion lat',
		4:'100.000 lat',
		5:'10.000 lat',
		6:'1000 lat',
		7:'100 lat',
		8:'10 lat',
		9:'rok',
		10:'miesiąc',
		11:'dzień',
		12:'godzina',
		13:'minuta',
		14:'sekunda',
        }
        return(datePrecision[prec])


class BasicBot(
    # Refer pywikobot.bot for generic bot classes
    SingleSiteBot,  # A bot only working on one site
    ConfigParserBot,  # A bot which reads options from scripts.ini setting file
    # CurrentPageBot,  # Sets 'current_page'. Process it in treat_page method.
    #                  # Not needed here because we have subclasses
    ExistingPageBot,  # CurrentPageBot which only treats existing pages
    NoRedirectPageBot,  # CurrentPageBot which only treats non-redirects
    AutomaticTWSummaryBot,  # Automatically defines summary; needs summary_key
):


    summary_key = 'basic-changing'

    def __init__(self, generator, **kwargs) -> None:
        """
        Initializer.

        @param generator: the page generator that determines on which pages
            to work
        @type generator: generator
        """
        # Add your own options to the bot and set their defaults
        # -always option is predefined by BaseBot class
        self.available_options.update({
            'replace': False,  # delete old text and write the new text
            'summary': None,  # your own bot summary
            'text': 'Test',  # add this text from option. 'Test' is default
            'top': False,  # append text on top of the page
            'outpage': u'Wikipedysta:mastiBot/test', #default output page
            'maxlines': 1000, #default number of entries per page
            'test': False, #test options
            'listscount': 1000, #max number of processed lists
            'labels': False, #show labels in WD Item
            'object': False, #show object content
            'testcount': 10000, #process only testcount items from input page
            'renew': False, #use original or generated pages as input
            'skip': 0, #skip that many first pages
        })

        # call initializer of the super class
        super().__init__(site=True, **kwargs)
        # assign the generator to the bot
        self.generator = generator


    def getLabel(self,ent,lang):
        #get entity label in lang language with fallback
        while ent.isRedirectPage():
            ent = ent.getRedirectTarget()
        entcontent = ent.get()
        for l in lang:
            try:
                #pywikibot.output('get label for:%s' % l)
                return(entcontent['labels'][l])
            except:
                continue
        return(None)

    def extractsection(self,page,section,level):
        # extract section of page returning it's content
        sectionR = re.compile(r'(?s)={'+str(level)+'}\s*?'+section+'\s*?={'+str(level)+'}(?P<text>.*?)\n={'+str(level)+'} ')
        if self.opt.test:
            pywikibot.output(u'(?s)={'+str(level)+'}\s*?'+section+'\s*?={'+str(level)+'}(?P<text>.*?)\n={'+str(level)+'} ')
        return(sectionR.search(page.text).group('text'))

    def genpages(self,text,ns=0, rgx=r'\[\[(?P<title>[^\|\]]*?)[\|\]]'):
        #generate pages based on wikilinks in text
        #titleR = re.compile(r'\[\[(?P<title>[^\|\]]*?)[\|\]]')
        titleR = re.compile(rgx)

        for t in titleR.finditer(text):
            title = re.sub(r'  ',' ',t.group('title'))
            page = pywikibot.Page(pywikibot.Site(),title)
            if page.namespace() == ns:
                yield page,t

    def header(self):
        header = u"Ta strona jest okresowo uaktualniana przez [[Wikipedysta:MastiBot|MastiBota]]. Ostatnia aktualizacja '''~~~~~'''."
        header += u"\n\nWszelkie uwagi proszę zgłaszać w [[Dyskusja wikipedysty:Masti|dyskusji operatora]]."
        header += u"\n:<small>Znalezione artykuły proszę wpisywać w kolumnie ''Link''</small>"
        header += u"\n:<small>Uwagi i komentarze kolumnie ''Uwagi''</small>"
        header += u"\n:<small>Pozostałę kolumny są aktualizowane automatycznie i zmiany w nich nie zostaną zachowane</small>"
        header +=u'\n\n{| class="wikitable sortable" style="font-size:85%;"'
        header +=u'\n|-'
        header +=u'\n!Nr'
        header +=u'\n!Link'
        header +=u'\n!Artykuł'
        header +=u'\n!Wikidane'
        header +=u'\n!Jest to'
        header +=u'\n!Data urodzenia'
        header +=u'\n!Data śmierci'
        header +=u'\n!Zajęcie'
        header +=u'\n!Opis'
        header +=u'\n!Uwagi'
        return(header)

    def footer(self):
        footer = u'\n|}'
        return(footer)

    def resetCounters(self):
        actionCounters['blue'] = 0
        actionCounters['red'] = 0
        actionCounters['redirs'] = 0
        actionCounters['disambigs'] = 0

    def run(self):

        reflinks = [] #initiate list
        count = 0
        for tpage in self.generator:
            count += 1
            if self.opt.test:
                pywikibot.output(u'[%s]Treating #%i: %s' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), count, tpage.title()))

            text = tpage.text

            if count <= int(self.opt.skip): 
                continue
            if count > int(self.opt.listscount):
                break
            if self.opt.test:
                pywikibot.output(u'[%s][%i]L:%s' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), count, tpage.title() ))
            try:
                refs = self.treat(tpage) 
            except pywikibot.exceptions.MaxlagTimeoutError:
                refs = None
                pywikibot.output('[%s] MaxlagTimeoutError in [[%s]]' % \
                    (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), tpage.title()))
            #if self.opt.test:
            #    pywikibot.output(refs)
            if refs:
                reflinks.append(refs)
            self.resetCounters()

        outputpage = self.opt.outpage

    def getData(self,page):
        #get data from page & WD
        #return Person object
        sex = { 'Q6581097':'mężczyzna', 'Q6581072':'kobieta','Q1097630':'obojnactwo', 'Q2449503':'transmężczyzna', 'Q1052281':'transkobieta', 'Q48270':'genderqueer'}

        obj = Person()
        obj.title = page.title()

        try:
            wd = pywikibot.ItemPage.fromPage(page)
            # while wd.ItemPage.isRedirectPage():
            #    pywikibot.output(('Processing WD redirect'))
            #    wd = wd.ItemPage.getRedirectTarget()
            wdcontent = wd.get()
            obj.wditem = '[[:d:%s]]' % wd.title()
            obj.wdexists = True
            if self.opt.labels:
                pywikibot.output(wdcontent['claims'].keys())
        except:
            pywikibot.output('WikiData page do not exists for: [[%s]]' % page.title())
            obj.wdexists = False
            return(obj)


        wbs = pywikibot.Site('wikidata','wikidata')

        for pid, claims in wdcontent['claims'].items():
            for claim in claims:
                trg = claim.getTarget()
                prp = pywikibot.PropertyPage(wbs,title=pid)

                prpcontent = prp.get()
                #pywikibot.output('Property content:%s' % self.getLabel(prp,['xhs','pl','sco','en']))
                #pywikibot.output('Property:%s Value:%s' % (pid,trg))
                if pid in ['P21','P569','P570','P106']:
                    if self.opt.labels:
                        pywikibot.output('P:%s, V:%s' % (self.getLabel(prp,['pl','en']),trg))
                if pid == 'P21':
                     try:
                         obj.sex = sex[trg.title()]
                     except:
                         obj.sex = 'inna'
                if pid == 'P31':
                     obj.instanceof = self.getLabel(trg,['pl','en'])
                if pid == 'P569':
                    obj.setDoB(trg)
                if pid == 'P570':
                    obj.setDoD(trg)
                if pid == 'P106':
                    lbl = self.getLabel(trg,['pl','en'])
                    if lbl:
                        obj.occupation.append(lbl)

        return(obj)

    def treat(self, page):
        result = []
        text = page.text
        count = 0

        obj = Person()

        if self.opt.test:
            pywikibot.output(u'Treat:[%s][%i][%i]:%s' % \
               (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), actionCounters['total'], count, page.title() ))
        if not page.exists():
            actionCounters['red'] += 1
        else:
            actionCounters['blue'] += 1
        pp = page
        while pp.isRedirectPage():
            actionCounters['redirs'] += 1
            pp = pp.getRedirectTarget()
            if self.opt.test:
                pywikibot.output(u'Redirect:[%s]:%s' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), pp.title() ))

        if pp.isDisambig():
            actionCounters['disambigs'] += 1
            obj = self.getData(pp)
            obj.instanceof = '<strona ujednoznaczniająca>'
            obj.disambig = True
            obj.title = pp.title()
            if self.opt.test:
                 pywikibot.output(u'Disambig:[%s][#%i]:%s' % \
                     (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),actionCounters['disambigs'], pp.title() ))
            else:
                obj = self.getData(pp)


        obj.link = page.title()

        if self.opt.object:
            obj.personPrint()
        result.append(obj)

        if self.opt.renew:
            self.generateresultspage(result,page.title(),self.header(),self.footer())
        else:
            self.generateresultspage(result,page.title()+'/Tabela',self.header(),self.footer())

    def generateresultspage(self, redirlist, pagename, header, footer):
        """
        Generates results page from redirlist
        Starting with header, ending with footer
        Output page is pagename
        """
        maxlines = int(self.opt.maxlines)
        finalpage = header

        res = redirlist
        itemcount = 0
        if self.opt.test:
            pywikibot.output(u'GENERATING RESULTS')
        for i in res:

            if self.opt.test:
                #pywikibot.output(i)
                i.personPrint()
            itemcount += 1
            if i.WDexists():
                if i.isDisambig():
                    finalpage += u'\n|-\n| %i || [[%s]] || [[%s]] || %s || colspan=4 style="background-color:NavajoWhite; text-align:center;" | %s || %s || %s' % \
                      ( itemcount, i.link, i.title, i.wditem, i.whatIs(), i.description, i.comment )
                else:
                    finalpage += u'\n|-\n| %i || [[%s]] || [[%s]] || %s || %s || %s || %s || %s || %s || %s' % \
                      ( itemcount, i.link, i.title, i.wditem, i.whatIs(), i.DoB(), i.DoD(), i.Occupation(), i.description, i.comment )
            else:
                finalpage += u'\n|-\n| %i || [[%s]] || [[%s]] || %s || colspan=4 style="background-color:LightSteelBlue; text-align:center;" | <brak danych> || %s || %s' % \
                      ( itemcount, i.link, i.title, i.wditem, i.description, i.comment )

        finalpage += footer 

        finalpage += '\n\n=== Statystyka ==='
        finalpage += '\n* Istniejące: %i' % actionCounters['blue']
        finalpage += '\n* Nieistniejące: %i' % actionCounters['red']
        finalpage += '\n* Przekierowania: %i' % actionCounters['redirs']
        finalpage += '\n* Ujednoznacznienia: %i' % actionCounters['disambigs']
        
        if self.opt.test:
            pywikibot.output(finalpage)
        success = True
        outpage = pywikibot.Page(pywikibot.Site(), pagename)
        outpage.text = finalpage

        if self.opt.test:
            pywikibot.output(outpage.title())
        
        outpage.save(summary=self.opt.summary)

        return(success)

def main(*args: Tuple[str, ...]) -> None:
    """
    Process command line arguments and invoke bot.

    If args is an empty list, sys.argv is used.

    @param args: command line arguments
    """
    options = {}
    # Process global arguments to determine desired site
    local_args = pywikibot.handle_args(args)

    # This factory is responsible for processing command line arguments
    # that are also used by other scripts and that determine on which pages
    # to work on.
    gen_factory = pagegenerators.GeneratorFactory()

    # Process pagegenerators arguments
    local_args = gen_factory.handle_args(local_args)

    # Parse your own command line arguments
    for arg in local_args:
        arg, sep, value = arg.partition(':')
        option = arg[1:]
        if option in ('summary', 'text', 'outpage', 'maxlines', 'listscount', 'testcount', 'skip'):
            if not value:
                pywikibot.input('Please enter a value for ' + arg)
            options[option] = value
        # take the remaining options as booleans.
        # You will get a hint if they aren't pre-defined in your bot class
        else:
            options[option] = True

    # The preloading option is responsible for downloading multiple
    # pages from the wiki simultaneously.
    gen = gen_factory.getCombinedGenerator(preload=True)
    if gen:
        # pass generator and private options to the bot
        bot = BasicBot(gen, **options)
        bot.run()  # guess what it does
    else:
        pywikibot.bot.suggest_help(missing_generator=True)


if __name__ == '__main__':
    main()