jpn <- read.table(stdin(),header=TRUE)
      DATE VALUE
2000-01-01   8.9
2001-01-01   9.1
2002-01-01   9.5
2003-01-01   9.6
2004-01-01   9.0
2005-01-01   8.1
2006-01-01   7.5
2007-01-01   7.5
2008-01-01   7.0
2009-01-01   8.9
2010-01-01   9.0
2011-01-01   8.1

albums <- read.csv("https://gwern.net/doc/touhou/2013-vgmdb.csv")
albums <- with(albums,
           data.frame(albumid,reldate,publisher,game,albumtitles))
albums <- albums[grepl("ouhou", albums$game),]; rm(albums$game)

tracks <- read.csv("2013-vgmdb-tracks.csv")
rm(tracks$tracklistid, tracks$trackid, tracks$disctitle, tracks$disc)
tracks$length[tracks$length==0] <- NA # 0 is the default in the VGMdb schema
tracks <- tracks[tracks$albumid %in% albums$albumid,]
touhou <- merge(tracks, albums)
rm(touhou$albumid, albums, tracks)
# deal with the 41 dates with the format "2005-09-00" (the 0th months or days are not real...)
touhou$date <- as.Date(sub("-00","-01",as.character(touhou$reldate)))
rm(touhou$reldate)
touhou$year <- as.integer(format(touhou$date, "%Y"))
# upcase, rearrange to torrent's order
colnames(touhou) <- c("Track","Title","Length","Circle","Album","Date","Year")
touhou <- touhou[c(2,7,5,3,1,4,6)]
write.csv(touhou, file="2013-vgmdb-touhou.csv", row.names=FALSE)

touhou <- read.csv("https://gwern.net/doc/touhou/2013-torrent.csv",
                    colClasses=c("character", "integer", "factor", "character",
                                         "integer", "factor", "character",
                                         "character", "numeric", "integer",
                                         "factor", "Date"))
summary(touhou$Year)
perYear <- table(touhou$Year); perYear; plot(perYear)
#
#  2002  2003  2004  2005  2006  2007  2008  2009  2010  2011  2012
#    13    23   255  1241  2070  2599  5073 10278  9765  7494  2999

runningTotal <- cumsum(table(touhou$Year)); runningTotal; plot(runningTotal)
# 2002  2003  2004  2005  2006  2007  2008  2009  2010  2011  2012
#   13    36   291  1532  3602  6201 11274 21552 31317 38811 41810

## FileSize is in megabytes, so gigabytes
sum(touhou[touhou$Year==2012,]$FileSize, na.rm=TRUE) / 1000
# [1] 19.5

import Text.HTML.TagSoup (fromAttrib, fromTagText, isTagOpen, isTagText,
                          parseTags, (~/=), Tag(TagComment, TagOpen,TagText))
import Network.HTTP (getResponseBody, getRequest, simpleHTTP)
import Data.List (isInfixOf, isPrefixOf, nub, sort, stripPrefix)
import Data.Char (isSpace)
import Codec.Binary.UTF8.String (decodeString)
import Data.Maybe (fromMaybe, isJust, listToMaybe)
import Data.List.Split (keepDelimsL, split, whenElt)
import Control.Monad (join, unless)

main :: IO ()
main = do albums1 <- getAlbums "https://en.touhouwiki.net/wiki/List_by_Groups"
          albums2 <- getAlbums "https://en.touhouwiki.net/wiki/List_of_Old_Touhou_Arrangement_Groups"
          let albumURLs = map ("https://en.touhouwiki.net"++) $ nub $ sort $ albums1 ++ albums2
          putStrLn header
          mapM_ getAlbum albumURLs

getAlbums :: String -> IO [String]
getAlbums index = do touhou <- openURL index
                     return $ drop 22 $ reverse [link | (TagOpen "a" ((ref, link):_)) <- parseTags touhou,
                                                        ref=="href",
                                                        "/wiki/" `isPrefixOf` link,
                                                        let fltr x = not (x `isInfixOf` link),
                                                        fltr "_Groups", fltr "Touhou_Wiki:", fltr "Special:",
                                                        fltr "Template:", fltr "Category:", fltr "Talk:"]

type Album = [Track]
data Track = Track { title :: String, year :: Int, date :: String,artist :: String,
                     album :: String, event :: String, circle :: String, duration :: Maybe Int,
                     track :: Int } deriving Show
empty :: Track
empty = Track {title="",year=0,date="",album="",event="",circle="",artist="",duration=Nothing,track=0}

header :: String
header = "Title,Year,Date,Album,Event,Circle,Duration,Track"
convert :: Track -> String
convert t = "\"" ++ dequote (title t) ++ "\"," ++ show (year t) ++ "," ++ show (date t) ++ ",\"" ++ dequote (album t) ++ "\",\"" ++
             event t ++ "\",\"" ++ circle t ++ "\"," ++ maybe "" show (duration t) ++ "," ++ show (track t)

-- You are not expected to understand this.
getAlbum :: String -> IO ()
getAlbum a = do t <- fmap parseTags $ openURL a
                --TagText "Released",TagClose "th",TagOpen "td" [],TagText "\n2007-03-23"
                let dt = let target = dropWhile (TagText "Released" ~/=) t
                         in if null target then ""
                            else deleteParens $ fromTagText $ head $ tail $ filter isTagText target
                -- TagText "Released",TagClose "th",TagOpen "td" [],TagText "\n2009/02/08 (",
                -- TagOpen "a" [("href","/wiki/Category:Sunshine_Creation_42"),("title","Category:Sunshine Creation 42")],
                -- TagText "Sunshine Creation 42",TagClose "a",TagText ")",TagClose "td"
                let evnt = let stream = filter isTagText $ dropWhile (TagText "Released" ~/=) t
                           in if '(' == last (fromTagText $ head $ take 5 $ drop 1 stream) -- )
                              then fromTagText (filter isTagText (dropWhile (TagText "Released" ~/=) t) !! 2)
                              else ""
                -- "2007-03-23"
                let yr = if null dt then 0 else read (take 4 dt)::Int
                -- TagText "Album by CODE ZTS LABEL"
                let hasCrcl = [cl | TagText cl <- t, "Album by " `isPrefixOf` cl]
                unless (null hasCrcl || (yr==0 && null dt)) $ do
                    let crcl = lookForCircle t
                    -- TagText "Selfregards2 - Touhou Wiki - Characters, games, locations, and more"
                    let albm = reverse $ drop 55 $ reverse $ head [al | TagText al <- t,
                                 " - Touhou Wiki - Characters, games, locations, and more" `isInfixOf` al]
                    let dflt = empty { date = dt, year = yr, circle = crcl, album = albm, event = evnt }
                    let table = filter (\x -> not ("Disc" `isInfixOf` x || " CD" `isInfixOf` x)) $
                                 filter (not . all isSpace) $
                                  map (trim . fromTagText) $ filter isTagText $
                                   drop 5 $ takeWhile (TagOpen "table"
                                     [("class","navbox"),("cellspacing","0"),
                                      ("style","background:#FFFBEE;border-color:#A8A077;")] ~/=) $
                                       takeWhile (TagComment "" ~/=) $
                                        dropWhile (TagOpen "span"
                                         [("class","mw-headline"),("id","Tracks")] ~/=) t
                    let tracks = filter (not . null) $
                                  split (keepDelimsL $ whenElt
                                   (\x -> length x==3 && "." `isInfixOf` x &&  isJust(maybeRead x :: Maybe Int))) table
                    mapM_ (putStrLn . convert . trackToTrack dflt) tracks

-- TagText "Album by ",TagOpen "a"
-- [("href","/wiki/ALiCE%27S_EMOTiON"),("title","ALiCE'S EMOTiON")],TagText
-- "ALiCE'S EMOTiON",TagClose "a"
lookForCircle :: [Tag String] -> String
lookForCircle t = let c = head [cl | TagText cl <- t, "Album by " `isPrefixOf` cl]
                      res = if c == "Album by " then (let tg = (dropWhile (TagText "Album by " ~/=) t !! 2)
                        in if isTagText tg then fromTagText tg else
                            (if isTagOpen tg then fromAttrib "title" tg else "") ) else drop 9 c
                      in if "(page does not exist)" `isInfixOf` res then takeWhile (/='(') res else res -- )

-- ["01.","The mom","(04:07)","arrangement: ZTS","composition: ZTS","original title: The mom","source: Parhelia"]
trackToTrack :: Track -> [String] -> Track
trackToTrack tr t = tr { track = fromMaybe 0 (maybeRead (head t) :: Maybe Int),
                         title = t !! 1,
                         duration = if length t >=3 then Just (timeConverter $ deleteParens (t !! 2)) else Nothing,
                         artist =  lookForAnArtist t }

lookForAnArtist :: [String] -> String
lookForAnArtist t = let targets = dropWhile (\x -> not ("arrangement:" `isPrefixOf` x || "composition:" `isPrefixOf` x)) t
                        target
                            | null targets = ""
                            | last (head targets) == ':' = head targets ++ (targets !! 1)
                            | otherwise = head targets
                    in trim $ fromMaybe "" $ join $ fmap (stripPrefix "arrangement:") $ stripPrefix "composition:" target

-- utility functions
openURL :: String -> IO String
openURL url = fmap decodeString (simpleHTTP (getRequest url) >>= getResponseBody)
deleteParens, trim, dequote :: String -> String
deleteParens = trim . filter (\x -> x /= '(' && x /= ')')
trim = reverse . dropWhile isSpace . reverse . dropWhile isSpace
dequote = map (\x -> if x=='"' then '\'' else x) -- "')
timeConverter :: String -> Int
timeConverter n = let (m,s) = break (==':') n
                      m' = maybeRead m :: Maybe Int
                      s' = maybeRead (drop 1 s) :: Maybe Int
                  in (fromMaybe 0 m' * 60) + fromMaybe 0 s'
maybeRead :: Read a => String -> Maybe a
maybeRead = fmap fst . listToMaybe . reads

import Text.HTML.TagSoup (fromTagText, isTagOpenName, isTagText, Tag(TagOpen,TagText), parseTags)
import Network.HTTP (getResponseBody, getRequest, simpleHTTP)
import Data.List (isPrefixOf, sort)
import Data.Char (isSpace)
import Codec.Binary.UTF8.String (decodeString)

main :: IO ()
main = do albumsURLs <- getAlbums
          albums <- mapM openURL (sort albumsURLs)
          let metadata = map toAlbum albums
          writeFile "vgmdb.csv" $ unlines (header : concatMap (map convert) metadata)

type Album = [Track]
data Track = Track { title :: String,
                     year :: Int,
                     date :: String,
                     album :: String,
                     circle :: String,
                     duration :: Maybe Int,
                     track :: Int } deriving Show
empty :: Track
empty = Track {title="",year=0,date="",album="",circle="",duration=Nothing,track=0}
header :: String
header = "Title,Year,Date,Album,Circle,Duration,Track"
convert :: Track -> String
convert t = "\"" ++ title t ++ "\"," ++ show(year t) ++ "," ++ show (date t) ++ ",\"" ++
              album t ++ "\",\"" ++ circle t ++ "\"," ++ maybe "" show(duration t) ++ "," ++ show (track t)

-- example album link: 'TagOpen "a" [("class","albumtitle album-doujin"),
--                                   ("href","https://vgmdb.net/album/36901"),
--                                   ("title","Majo to Ringo to Samayou Kimi to")]'
getAlbums :: IO [String]
getAlbums = do touhou <- openURL "https://vgmdb.net/product/9"
               return [snd(atts !! 1) | TagOpen "a" atts <- parseTags touhou, snd(head atts)=="albumtitle album-doujin"]

-- need 'decodeString' to deal with Japanese glyphs; see https://stackoverflow.com/questions/10558003/how-to-get-utf8-rss-feed
openURL :: String -> IO String
openURL url = fmap decodeString (simpleHTTP (getRequest url) >>= getResponseBody)

toAlbum :: String -> Album
toAlbum page = let tags = parseTags page
                   (yr,dt) = extractDate tags
                   albm = extractAlbum tags
                   crcl = extractCircle tags
                   files = extractMusic tags
               in map (\t -> Track {title = title t, year = yr, date = dt,
                                    album = albm, circle = crcl, duration = duration t,
                                    track = track t}) files


-- TagOpen "a" [("title","View albums released on Dec 30, 2011"),("href","/db/calendar.php?year=2011&month=12#20111230")]
extractDate :: [Tag String] -> (Int,String)
extractDate t = let (a:b:_) = map snd $ head [atts | TagOpen "a" atts <- t,
                                       let ttle = snd(head atts),
                                       "View albums released on " `isPrefixOf` ttle]
        in (read(reverse $ take 4 $ reverse a)::Int,
           tail$ snd $ break (=='#') b)

-- TagOpen "title" [],TagText "Gensou Rashinban - VGMdb",TagClose "title",
extractAlbum :: [Tag String] -> String
extractAlbum t = (\(TagText x) -> reverse $ drop 8 $ reverse x) (dropWhile (not . isTagOpenName "title") t !! 1)

-- [TagText "Published by",TagClose "b",TagClose "span",TagClose "td",TagText "\r\n",
-- TagOpen "td" [],TagOpen "a" [("href","/org/217")],TagOpen "span"
-- [("class","productname"),("lang","en"),("style","display:inline")],TagText "PopKorn"]
extractCircle :: [Tag String] -> String
extractCircle t = fromTagText(head (drop 8 (dropWhile (\x -> not(isTagText x && (fromTagText x)=="Published by")) t)))

extractMusic :: [Tag String] -> [Track]
extractMusic t = let tracks = filter (not . all isSpace) $
                               tail $ dropWhile (\z -> z /= "Disc 1") $
                                map fromTagText $ filter isTagText $
                                 takeWhile (\y -> not(isTagText y && (fromTagText y)=="Disc length")) $
                                  dropWhile (\x -> not(isTagText x && (fromTagText x)=="Tracklist")) t
                   in if length tracks `rem` 3 == 0 then threezip tracks else twozip tracks
       where
       twozip,threezip :: [String] -> [Track]
       threezip [] = []
       threezip (a:b:c:d) = empty {title=b,duration=Just (timeConverter c),track=read a} : threezip d
       threezip _ = []
       twozip [] = []
       twozip (a:b:d) = empty {title=b,duration=Nothing,track=read a} : twozip d
       twozip _ = []
       timeConverter :: String -> Int
       timeConverter n = let (m,s) = break (==':') n in ((read m :: Int) * 60) + (read (tail s) :: Int)