src/lyrics_fetcher.cpp

   1 /***************************************************************************
   2  *   Copyright (C) 2008-2016 by Andrzej Rybczak                            *
   3  *   electricityispower@gmail.com                                          *
   4  *                                                                         *
   5  *   This program is free software; you can redistribute it and/or modify  *
   6  *   it under the terms of the GNU General Public License as published by  *
   7  *   the Free Software Foundation; either version 2 of the License, or     *
   8  *   (at your option) any later version.                                   *
   9  *                                                                         *
  10  *   This program is distributed in the hope that it will be useful,       *
  11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13  *   GNU General Public License for more details.                          *
  14  *                                                                         *
  15  *   You should have received a copy of the GNU General Public License     *
  16  *   along with this program; if not, write to the                         *
  17  *   Free Software Foundation, Inc.,                                       *
  18  *   51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.              *
  19  ***************************************************************************/
  20
  21 #include "config.h"
  22 #include "curl_handle.h"
  23
  24 #ifdef HAVE_CURL_CURL_H
  25
  26 #include <cstdlib>
  27 #include <cstring>
  28 #include <boost/algorithm/string/join.hpp>
  29 #include <boost/algorithm/string/replace.hpp>
  30 #include <boost/algorithm/string/split.hpp>
  31 #include <boost/algorithm/string/trim.hpp>
  32 #include <boost/regex.hpp>
  33
  34 #include "charset.h"
  35 #include "lyrics_fetcher.h"
  36 #include "utility/html.h"
  37 #include "utility/string.h"
  38
  39 std::unique_ptr<LyricsFetcher> toLyricsFetcher(const std::string &s)
  40 {
  41         if (s == "lyricwiki")
  42                 return std::make_unique<LyricwikiFetcher>();
  43         else if (s == "azlyrics")
  44                 return std::make_unique<AzLyricsFetcher>();
  45         else if (s == "genius")
  46                 return std::make_unique<GeniusFetcher>();
  47         else if (s == "sing365")
  48                 return std::make_unique<Sing365Fetcher>();
  49         else if (s == "lyricsmania")
  50                 return std::make_unique<LyricsmaniaFetcher>();
  51         else if (s == "metrolyrics")
  52                 return std::make_unique<MetrolyricsFetcher>();
  53         else if (s == "justsomelyrics")
  54                 return std::make_unique<JustSomeLyricsFetcher>();
  55         else if (s == "tekstowo")
  56                 return std::make_unique<TekstowoFetcher>();
  57         else if (s == "internet")
  58                 return std::make_unique<InternetLyricsFetcher>();
  59         throw std::runtime_error("no lyrics fetcher named '" + s + "'");
  60 }
  61
  62 const char LyricsFetcher::msgNotFound[] = "Not found";
  63
  64 LyricsFetcher::Result LyricsFetcher::fetch(const std::string &artist, const std::string &title)
  65 {
  66         Result result;
  67         result.first = false;
  68
  69         std::string url = urlTemplate();
  70         boost::replace_all(url, "%artist%", artist);
  71         boost::replace_all(url, "%title%", title);
  72
  73         std::string data;
  74         CURLcode code = Curl::perform(data, url);
  75
  76         if (code != CURLE_OK)
  77         {
  78                 result.second = curl_easy_strerror(code);
  79                 return result;
  80         }
  81
  82         auto lyrics = getContent(regex(), data);
  83
  84         if (lyrics.empty() || notLyrics(data))
  85         {
  86                 result.second = msgNotFound;
  87                 return result;
  88         }
  89
  90         data.clear();
  91         for (auto it = lyrics.begin(); it != lyrics.end(); ++it)
  92         {
  93                 postProcess(*it);
  94                 if (!it->empty())
  95                 {
  96                         data += *it;
  97                         if (it != lyrics.end()-1)
  98                                 data += "\n\n----------\n\n";
  99                 }
 100         }
 101
 102         result.second = data;
 103         result.first = true;
 104         return result;
 105 }
 106
 107 std::vector<std::string> LyricsFetcher::getContent(const char *regex_, const std::string &data)
 108 {
 109         std::vector<std::string> result;
 110         boost::regex rx(regex_);
 111         auto first = boost::sregex_iterator(data.begin(), data.end(), rx);
 112         auto last = boost::sregex_iterator();
 113         for (; first != last; ++first)
 114                 result.push_back(first->str(1));
 115         return result;
 116 }
 117
 118 void LyricsFetcher::postProcess(std::string &data) const
 119 {
 120         data = unescapeHtmlUtf8(data);
 121         stripHtmlTags(data);
 122         // Remove indentation from each line and collapse multiple newlines into one.
 123         std::vector<std::string> lines;
 124         boost::split(lines, data, boost::is_any_of("\n"));
 125         for (auto &line : lines)
 126                 boost::trim(line);
 127         std::unique(lines.begin(), lines.end(), [](std::string &a, std::string &b) {
 128                 return a.empty() && b.empty();
 129         });
 130         data = boost::algorithm::join(lines, "\n");
 131         boost::trim(data);
 132 }
 133
 134 /***********************************************************************/
 135
 136 LyricsFetcher::Result LyricwikiFetcher::fetch(const std::string &artist, const std::string &title)
 137 {
 138         LyricsFetcher::Result result = LyricsFetcher::fetch(artist, title);
 139         if (result.first == true)
 140         {
 141                 result.first = false;
 142
 143                 std::string data;
 144                 CURLcode code = Curl::perform(data, result.second, "", true);
 145
 146                 if (code != CURLE_OK)
 147                 {
 148                         result.second = curl_easy_strerror(code);
 149                         return result;
 150                 }
 151
 152                 auto lyrics = getContent("<div class='lyricbox'>(.*?)</div>", data);
 153
 154                 if (lyrics.empty())
 155                 {
 156                         result.second = msgNotFound;
 157                         return result;
 158                 }
 159                 std::transform(lyrics.begin(), lyrics.end(), lyrics.begin(), unescapeHtmlUtf8);
 160                 bool license_restriction = std::any_of(lyrics.begin(), lyrics.end(), [](const std::string &s) {
 161                         return s.find("Unfortunately, we are not licensed to display the full lyrics for this song at the moment.") != std::string::npos;
 162                 });
 163                 if (license_restriction)
 164                 {
 165                         result.second = "Licence restriction";
 166                         return result;
 167                 }
 168
 169                 data.clear();
 170                 for (auto it = lyrics.begin(); it != lyrics.end(); ++it)
 171                 {
 172                         stripHtmlTags(*it);
 173                         boost::trim(*it);
 174                         if (!it->empty())
 175                         {
 176                                 data += *it;
 177                                 if (it != lyrics.end()-1)
 178                                         data += "\n\n----------\n\n";
 179                         }
 180                 }
 181
 182                 result.second = data;
 183                 result.first = true;
 184         }
 185         return result;
 186 }
 187
 188 bool LyricwikiFetcher::notLyrics(const std::string &data) const
 189 {
 190         return data.find("action=edit") != std::string::npos;
 191 }
 192
 193 /**********************************************************************/
 194
 195 LyricsFetcher::Result GoogleLyricsFetcher::fetch(const std::string &artist, const std::string &title)
 196 {
 197         Result result;
 198         result.first = false;
 199
 200         std::string search_str = artist;
 201         search_str += "+";
 202         search_str += title;
 203         search_str += "+%2B";
 204         search_str += siteKeyword();
 205
 206         std::string google_url = "http://www.google.com/search?hl=en&ie=UTF-8&oe=UTF-8&q=";
 207         google_url += search_str;
 208         google_url += "&btnI=I%27m+Feeling+Lucky";
 209
 210         std::string data;
 211         CURLcode code = Curl::perform(data, google_url, google_url);
 212
 213         if (code != CURLE_OK)
 214         {
 215                 result.second = curl_easy_strerror(code);
 216                 return result;
 217         }
 218
 219         auto urls = getContent("<A HREF=\"(.*?)\">here</A>", data);
 220
 221         if (urls.empty() || !isURLOk(urls[0]))
 222         {
 223                 result.second = msgNotFound;
 224                 return result;
 225         }
 226
 227         data = unescapeHtmlUtf8(urls[0]);
 228
 229         URL = data.c_str();
 230         return LyricsFetcher::fetch("", "");
 231 }
 232
 233 bool GoogleLyricsFetcher::isURLOk(const std::string &url)
 234 {
 235         return url.find(siteKeyword()) != std::string::npos;
 236 }
 237
 238 /**********************************************************************/
 239
 240 bool MetrolyricsFetcher::isURLOk(const std::string &url)
 241 {
 242         // it sometimes return link to sitemap.xml, which is huge so we need to discard it
 243         return GoogleLyricsFetcher::isURLOk(url) && url.find("sitemap") == std::string::npos;
 244 }
 245
 246 /**********************************************************************/
 247
 248 LyricsFetcher::Result InternetLyricsFetcher::fetch(const std::string &artist, const std::string &title)
 249 {
 250         GoogleLyricsFetcher::fetch(artist, title);
 251         LyricsFetcher::Result result;
 252         result.first = false;
 253         result.second = "The following site may contain lyrics for this song: ";
 254         result.second += URL;
 255         return result;
 256 }
 257
 258 bool InternetLyricsFetcher::isURLOk(const std::string &url)
 259 {
 260         URL = url;
 261         return false;
 262 }
 263
 264 #endif // HAVE_CURL_CURL_H
 265