1. Python / Говнокод #23172

    0

    1. 01
    2. 02
    3. 03
    4. 04
    5. 05
    6. 06
    7. 07
    8. 08
    9. 09
    10. 10
    11. 11
    12. 12
    13. 13
    14. 14
    15. 15
    16. 16
    17. 17
    18. 18
    19. 19
    20. 20
    21. 21
    22. 22
    23. 23
    24. 24
    25. 25
    26. 26
    27. 27
    28. 28
    29. 29
    30. 30
    31. 31
    32. 32
    33. 33
    34. 34
    35. 35
    36. 36
    37. 37
    38. 38
    39. 39
    40. 40
    41. 41
    42. 42
    43. 43
    44. 44
    45. 45
    46. 46
    47. 47
    48. 48
    49. 49
    50. 50
    51. 51
    52. 52
    53. 53
    54. 54
    55. 55
    56. 56
    57. 57
    58. 58
    59. 59
    60. 60
    import csv
import json

import requests

from io import StringIO

from lxml import etree


class DayPicParser(object):
    url = 'урл_до_сайта'

    def get_info(self):
        handler = open('daypic.csv', 'w+')
        page = 1
        rows = []
        tags = set()
        while page <= 1077:
            response = requests.get('{}/page/{}'.format(self.url, page))
            response.connection.close()
            parser = etree.HTMLParser()
            tree = etree.parse(StringIO(response.text), parser)
            posts = tree.xpath('//div[@class="posts"]/div[@class="post"]')
            for i, post in enumerate(posts):
                title = post.find('./h2/a')
                link_to_fulltext = title.attrib['href']

                short_text = post.find('./div[@class="text"]//p[1]')
                response = requests.get(link_to_fulltext)
                full_parser = etree.HTMLParser()
                full_tree = etree.parse(StringIO(response.text), full_parser)
                full_post = full_tree.xpath('//div[@class="text"]//p')

                main_image = None
                full_text = []
                for inner in full_post:
                    if inner.find('.//noindex') is not None:
                        continue
                    if inner.find('.//img') is not None:
                        src = inner.find('.//img').attrib['src']
                        if not main_image:
                            main_image = src
                        full_text.append({'image': src})
                    for paragraph in etree.tostring(inner, method="text", encoding='utf-8').decode('utf-8').strip().split("\n"):
                        if paragraph:
                            full_text.append({'text': paragraph})
                rows.append({'title': title.text.strip(),
                             'preview': main_image if main_image else '',
                             'short_text': short_text.text.strip(),
                             'full_text': full_text})
                print('Post {} of {}'.format(i, len(posts)))
            page += 1
            print(page)
        handler.write(json.dumps(rows))
        handler.close()
        print(tags)


DayPicParser().get_info()

    Запостил: storvus, 10 Июля 2017

    Комментарии (6) RSS

    Добавить комментарий