OmegaChart

Fork

// Copyright (c) 2014 panacoran <panacoran@users.sourceforge.jp>
// This program is part of OmegaChart.
// OmegaChart is licensed under the Apache License, Version 2.0.

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading;
using Zanetti.Data;

namespace Zanetti.DataSource.Specialized
{
    internal class YahooDataSource : DailyDataSource
    {
        private Queue<int> _codeQueue;
        private readonly List<int> _codes = new List<int>();
        private readonly List<int> _series = new List<int>();
        private const int DaysAtOnce = 20; // 一度に取得する時系列の営業日数
        private static Random _rng = new Random();

        private class FetchResult
        {
            public enum Status
            {
                Success,
                Failure,
                Obsolete,
                Retry
            }

            public int Code;
            public SortedDictionary<int, NewDailyData> Prices;
            public Status ReturnStatus;
        }

        public YahooDataSource(int[] dates) : base(dates)
        {
            foreach (AbstractBrand brand in Env.BrandCollection.Values)
            {
                var basic = brand as BasicBrand;
                if (brand.Market == MarketType.B || brand.Market == MarketType.Custom ||
                    basic == null || basic.Obsolete)
                    continue;
                _codes.Add(brand.Code);
            }
        }

        public override int TotalStep =>
            (_codes.Count + 2) * ((_dates.Length + DaysAtOnce - 1) / DaysAtOnce); // +2はNikkei225とTOPIX

        public override void Run()
        {
            var dates = new List<int>(_dates);
            do
            {
                // 日経平均の時系列データの存在を確認する。
                var n = Math.Min(DaysAtOnce, dates.Count);
                var original = dates.GetRange(0, n);
                var nikkei225 = FetchPrices((int)BuiltInIndex.Nikkei225, original);
                if (nikkei225.ReturnStatus != FetchResult.Status.Success)
                    throw new Exception(
                        $"株価の取得に失敗しました。時間を置いて再試行してください。: {original[0]}～{original[original.Count - 1]}");
                dates.RemoveRange(0, n);
                _series.Clear();
                foreach (var date in original)
                {
                    if (nikkei225.Prices[date].close == 0)
                        nikkei225.Prices.Remove(date);
                    else
                        _series.Add(date);
                }
                if (_series.Count == 0)
                    return;
                UpdateDataFarm((int)BuiltInIndex.Nikkei225, nikkei225.Prices);
                SendMessage(AsyncConst.WM_ASYNCPROCESS, (int)BuiltInIndex.Nikkei225,
                    AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
                _codeQueue = new Queue<int>(_codes.OrderBy(x => _rng.Next()));
                _codeQueue.Enqueue((int)BuiltInIndex.TOPIX);
                var retry = 0;
                while (true)
                {
                    var numCodes = _codeQueue.Count;
                    for (var i = 0; i < numCodes; i++)
                    {
                        var result = FetchPrices(_codeQueue.Dequeue(), _series);
                        switch (result.ReturnStatus)
                        {
                            case FetchResult.Status.Failure:
                            case FetchResult.Status.Obsolete:
                                continue;
                            case FetchResult.Status.Retry:
                                _codeQueue.Enqueue(result.Code);
                                continue;
                        }
                        UpdateDataFarm(result.Code, result.Prices);
                        SendMessage(AsyncConst.WM_ASYNCPROCESS, result.Code, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
                        Thread.Sleep(_rng.Next(0, 500));
                    }
                    if (_codeQueue.Count == 0)
                        break;
                    if (retry++ == 10)
                        throw new Exception(
                            $"株価の取得に失敗しました。時間を置いて再試行してください。: {_series[0]}～{_series[_series.Count - 1]}");
                    Thread.Sleep(10000);
                }
            } while (dates.Count > 0);
        }

        public void UpdateDataFarm(int code, SortedDictionary<int, NewDailyData> prices)
        {
            var farm = (DailyDataFarm)Env.BrandCollection.FindBrand(code).CreateDailyFarm(prices.Count);
            foreach (var pair in prices)
                farm.UpdateDataFarm(pair.Key, pair.Value);
            farm.Save(Util.GetDailyDataFileName(code));
        }

        private FetchResult FetchPrices(int code, IList<int> dates)
        {
            var status = GetPage(code, Util.IntToDate(dates[0]), Util.IntToDate(dates[dates.Count - 1]), out var page);
            if (status == FetchResult.Status.Failure || status == FetchResult.Status.Retry)
                return new FetchResult {Code = code, ReturnStatus = status};
            return ParsePage(code, page, dates);
        }

        private FetchResult.Status GetPage(int code, DateTime begin, DateTime end, out string page)
        {
            string codeString;
            switch (code)
            {
                case (int)BuiltInIndex.Nikkei225:
                    codeString = "998407.O";
                    break;
                case (int)BuiltInIndex.TOPIX:
                    codeString = "998405.T";
                    break;
                default:
                    codeString = code.ToString();
                    break;
            }
            var oldUrl = $"https://info.finance.yahoo.co.jp/history/?code={codeString}&sy={begin.Year}&sm={begin.Month}&sd={begin.Day}&ey={end.Year}&em={end.Month}&ed={end.Day}&tm=d";
            var url = $"https://finance.yahoo.co.jp/quote/{codeString}.T/history?from={begin:yyyyMMdd}&to={end:yyyyMMdd}&timeFrame=d&page=1";
            page = null;
            retry:
            try
            {
                using (var reader = new StreamReader(Util.HttpDownload(url)))
                {
                    page = reader.ReadToEnd();
                }
            }
            catch (WebException e)
            {
                switch (e.Status)
                {
                    case WebExceptionStatus.ProtocolError:
                        switch (((HttpWebResponse)e.Response).StatusCode)
                        {
                            case (HttpStatusCode)999:
                            case HttpStatusCode.InternalServerError:
                            case HttpStatusCode.BadGateway:
                                return FetchResult.Status.Retry;
                            case HttpStatusCode.NotFound:
                                if (url == oldUrl)
                                    return FetchResult.Status.Failure;
                                url = oldUrl;
                                goto retry;
                        }
                        throw;
                    case WebExceptionStatus.Timeout:
                    case WebExceptionStatus.ConnectionClosed:
                    case WebExceptionStatus.ReceiveFailure:
                    case WebExceptionStatus.ConnectFailure:
                        return FetchResult.Status.Retry;
                    default:
                        throw;
                }
            }
            return FetchResult.Status.Success;
        }

        private static readonly Regex Valid = new Regex(
            @"<tr[^>]*><t[hd][^>]*>(?<year>\d{4})年(?<month>1?\d)月(?<day>\d?\d)日<\/t[hd]><td[^>]+>(?:<span[^>]+>)+(?<open>[0-9,.]+)(?:<\/span>)+<\/td><td[^>]+>(?:<span[^>]+>)+(?<high>[0-9,.]+)(?:<\/span>)+<\/td><td[^>]+>(?:<span[^>]+>)+(?<low>[0-9,.]+)(?:<\/span>)+<\/td><td[^>]+>(?:<span[^>]+>)+(?<close>[0-9,.]+)(?:<\/span>)+<\/td>(?:<td[^>]+>(?:<span[^>]+>)+(?<volume>[0-9,.]+)(?:<\/span>)+<\/td>.+?<\/td>)?<\/tr>",
            RegexOptions.Compiled);

        private static readonly Regex NoData = new Regex("時系列情報がありません");

        private static readonly Regex ValidOld = new Regex(
            @"<td>(?<year>\d{4})年(?<month>1?\d)月(?<day>\d?\d)日</td>" +
            "<td>(?<open>[0-9,.]+)</td><td>(?<high>[0-9,.]+)</td><td>(?<low>[0-9,.]+)</td>" +
            "<td>(?<close>[0-9,.]+)</td>(?:<td>(?<volume>[0-9,]+)</td>)?", RegexOptions.Compiled);

        private static readonly Regex NoDataOld = new Regex("該当する期間のデータはありません。<br>期間をご確認ください。");

        private static readonly Regex Obs =
            new Regex("該当する銘柄はありません。<br>再度銘柄（コード）を入力し、「表示」ボタンを押してください。", RegexOptions.Compiled);

        private static readonly Regex Empty = new Regex("<dl class=\"stocksInfo\">\n<dt></dt><dd class=\"category yjSb\"></dd>", RegexOptions.Compiled);

        private FetchResult ParsePage(int code, string buf, IEnumerable<int> dates)
        {
            if (buf == null)
                return null;
            var dict = new SortedDictionary<int, NewDailyData>();
            MatchCollection matches;

            matches = Valid.Matches(buf);
            if (matches.Count == 0)
            {
                if (!NoData.IsMatch(buf))
                {
                    matches = ValidOld.Matches(buf);
                    if (matches.Count == 0)
                    {
                        if (Obs.Match(buf).Success || Empty.Match(buf).Success) // 上場廃止(銘柄データが空のこともある)
                            return new FetchResult {ReturnStatus = FetchResult.Status.Obsolete};
                        if (!NoDataOld.IsMatch(buf))
                            throw new Exception("ページから株価を取得できません。");
                        // ここに到達するのは出来高がないか株価が用意されていない場合
                    }
                }
            }
            try
            {
                var shift = IsIndex(code) ? 100 : 10; // 指数は100倍、株式は10倍で記録する
                const NumberStyles s = NumberStyles.AllowDecimalPoint | NumberStyles.AllowThousands;
                foreach (Match m in matches)
                {
                    var date = new DateTime(int.Parse(m.Groups["year"].Value),
                        int.Parse(m.Groups["month"].Value),
                        int.Parse(m.Groups["day"].Value));
                    dict[Util.DateToInt(date)] = new NewDailyData
                    {
                        open = (int)(double.Parse(m.Groups["open"].Value, s) * shift),
                        high = (int)(double.Parse(m.Groups["high"].Value, s) * shift),
                        low = (int)(double.Parse(m.Groups["low"].Value, s) * shift),
                        close = (int)(double.Parse(m.Groups["close"].Value, s) * shift),
                        volume = m.Groups["volume"].Value == "" ? 0 : (int)double.Parse(m.Groups["volume"].Value, s)
                    };
                }
            }
            catch (FormatException e)
            {
                throw new Exception("ページから株価を取得できません。", e);
            }
            // 出来高がない日の株価データがないので値が0のデータを補う。
            foreach (var date in dates)
            {
                if (!dict.ContainsKey(date))
                    dict[date] = new NewDailyData();
            }
            return new FetchResult {Code = code, Prices = dict, ReturnStatus = FetchResult.Status.Success};
        }

        private bool IsIndex(int code)
        {
            return code == (int)BuiltInIndex.Nikkei225 ||
                code == (int)BuiltInIndex.TOPIX;
        }
    }
}