我喜欢这个叫做Keyforge的新纸牌游戏，我是一名业余数据科学家，希望为社区提供一些很棒的工具和功能。我的最终目标是开发一个应用程序，该应用程序可以扫描新的新甲板的QR码，并在现场告诉您新甲板与世界上其他甲板相比有多好以及放置方式如何。所有这些都使用机器学习。
制作游戏的公司有一个API，可以从中提取所有数据，并且它是JSON对象。我相信这是一个命令的字典（我复制了下面的一些JSON文件供您确认我是对的）。我对JSON对象没有太多的经验，但是总体上我对python感到满意。
我正在尝试将数据（全部或部分数据）解析或提取到sqlite3数据库中。我对数据库还很陌生，所以我很努力。
除了对sqlite 3进行JSON处理外，我还需要将数据处理为学习算法所需的格式，但我想我可以应付。
我尝试了很多在这里和在线上找到的东西，但是都没有用。
下面是一个JSON文件的开头和中间部分，因此您可以了解数据的结构。我读到我需要先弄平它，并尝试了一些方法，但是到目前为止我还没有运气。 我的目标是要有一张桌子，其中每一行都是一个牌组（或学习算法的样本）。在下面，您可以看到几个套牌：“ The Old Jeffries博士”和“神秘的Gendry V. Rineity”。该表的列将是唯一的ID号（主键，自动递增等），基本上是1、2、3等，而其余的列将是“名称”， “扩展”，“ power_level”等。我真正需要的实际最小数据是id号（主键，自动增加），每个卡座中36张卡中的每张的“ card_number”（这是来自1到370，这是游戏中的纸牌），然后是“胜利”，“亏损”，“休闲赢”和“休闲亏损”列，这对于在良好的成功套牌上训练学习算法至关重要。
{
"count": 459959, #this is how many decks are live in the World
"data": [ #this contains the name of each deck and other data
{
"name": "Dr. \"The Old\" Jeffries",
"expansion": 341,
"power_level": 0,
"chains": 0,
"wins": 0,
"losses": 0,
"id": "ec86db52-e41e-4e6a-9f1a-a2d0e3d3277d",
"is_my_deck": false,
"notes": [],
"is_my_favorite": false,
"is_on_my_watchlist": false,
"casual_wins": 0,
"casual_losses": 0,
"_links": {
"houses": [
"Brobnar",
"Dis",
"Logos"
],
"cards": [ #this is the list of 36 cards for this deck
"d438faa9-7920-437a-8d1c-682fade5d350",
"d438faa9-7920-437a-8d1c-682fade5d350",
"0ef760a3-68b9-42a9-93fa-419ea171917b",
"68e2188c-4002-43d4-9fe1-0262df26c33f",
"68e2188c-4002-43d4-9fe1-0262df26c33f",
"3d650fe4-817a-4922-ba0f-297c1ebf816d",
"652c4e38-c4fa-4e30-8f8d-036e95249529",
"d4f666db-302f-43d0-b0af-bd03071f92ce",
"0c3231e1-1230-4e7d-890e-6d3149125de2",
"d792387f-8392-49b3-ad7c-ccaf7552256f",
"f2c71c05-7a23-4465-8a89-82ab8e258a68",
"e1312fbf-c297-4d9f-b403-2d892271de62",
"d2edea65-7c2f-487f-a6f4-f44a077c4a65",
"10715fd2-031a-47ca-9119-9b7b2ec1d2c0",
"10715fd2-031a-47ca-9119-9b7b2ec1d2c0",
"be492d70-5c87-441e-8223-79fb2bce85c9",
"d42dd1d0-3462-410f-b683-dd0768b84188",
"5607fecd-b90e-4e12-84bc-cb36d079117c",
"ea2a390e-e121-4cbd-96c5-2430cc600e81",
"96548d93-b318-40e3-9f5c-3297c8070ebd",
"17e9dbd4-53cb-4c75-bdad-48e1550ff1e7",
"916f271b-9928-437c-bfc4-d60d32af8c7c",
"9ed7d241-1ca3-4a2a-b067-bb44776f7d4b",
"b8343462-b5d7-48b0-9e3b-f020c5e73c55",
"1283215c-3ea2-4d2b-9af4-452d7c0d57d9",
"1283215c-3ea2-4d2b-9af4-452d7c0d57d9",
"448c1335-d45b-473e-b222-d71f31ba0292",
"448c1335-d45b-473e-b222-d71f31ba0292",
"45d564a2-fcc9-4baa-8dc8-8e1a0fe2a37a",
"5a521238-f524-48e3-b121-40c16e1f7610",
"57bccc52-b6a1-4d11-b9d9-6356d8ac279c",
"cd83ebe7-f961-4e5e-a00e-046d1be5e5d3",
"53f7d3ec-a65f-4b05-8c82-74f44a7bdc44",
"60f095d7-1816-4f14-88ec-04412ebde43b",
"60f095d7-1816-4f14-88ec-04412ebde43b",
"bec84d69-68f0-456c-a7bd-9f1e94d55a22"
]
}
},
{
"name": "Gendry V. Rineity, the Enigmatic",
"expansion": 341,
"power_level": 0,
"chains": 0,
"wins": 0,
"losses": 0,
"id": "48761c52-1bf0-4437-ba12-f0544237da4e",
"is_my_deck": false,
"notes": [],
"is_my_favorite": false,
"is_on_my_watchlist": false,
"casual_wins": 0,
"casual_losses": 0,
"_links": {
"houses": [
"Dis",
"Logos",
"Mars"
],
"cards": [
"699f06e3-e47b-4910-90b9-c67fac157d6e",
"10715fd2-031a-47ca-9119-9b7b2ec1d2c0",
"10715fd2-031a-47ca-9119-9b7b2ec1d2c0",
"aa73a693-e1e6-4097-8010-ddc820cc6d96",
"9644c85a-12a7-44ff-a8bb-877dddb46995",
"750a9323-9c07-4ae7-be5e-79367b4a2a8d",
"c5ed37f7-0d05-48bc-a595-4f25c0ec1e6d",
"f97316b0-75a4-45a4-8735-15e72cc1568c",
"916f271b-9928-437c-bfc4-d60d32af8c7c",
"3d6a02d0-b5c8-49be-93e4-dfdd5c1200eb",
"9152cbad-d83f-4ee4-9846-87cc60d185f1",
"b8343462-b5d7-48b0-9e3b-f020c5e73c55",
"8c763540-bb69-47aa-be43-4a8ace89864c",
"f51e8ec0-ab0e-46a8-a5f5-680039d6e664",
"1838fbaa-a062-4593-acbe-53ecfadfb5cc",
"1838fbaa-a062-4593-acbe-53ecfadfb5cc",
"2cb1f58c-5979-4d3a-ae86-9dadc6000288",
"2cb1f58c-5979-4d3a-ae86-9dadc6000288",
"2cb1f58c-5979-4d3a-ae86-9dadc6000288",
"448c1335-d45b-473e-b222-d71f31ba0292",
"3c5c1881-486c-4911-a3ce-497ef258e8ba",
"2ec5cbf6-3c41-41ef-9cb7-33a0601fd607",
"2ec5cbf6-3c41-41ef-9cb7-33a0601fd607",
"c08c91f0-043a-4a8a-8761-6080e9f46183",
"16168a85-bbfa-4e54-8c84-5ea02e2a7da1",
"cc44ca9a-6994-4897-9308-ff332cc8de57",
"19b74b4e-bec8-4fbb-bd35-cb635f500249",
"0cc7c1ea-5196-40ff-b408-f31997c8ab4d",
"179f877a-9b59-46d6-a43e-15b4524af3c6",
"0e5e8a55-ab05-44be-8637-8362974dad8b",
"aabeebf7-1da5-4149-afab-e7e221b47d93",
"a8a3578c-7a61-4e15-90ac-483daf2aff16",
"1a84631d-7fcb-4c9a-a50c-9539dcb84928",
"211c5213-7838-4292-b9c4-fb3a663898ee",
"ac8fb9f6-ee8e-4434-85e8-d084a66c50db",
"ff104cf4-f99d-4021-a570-dd949e559e97"
]
}
},
#...
#... there are 25 decks per page. I can only get that many per API call
#...
"_linked": { #here are the actual houses per deck and the actual cards
"houses": [
{
"id": "Brobnar",
"name": "Brobnar",
"image": "https://cdn.keyforgegame.com/media/houses/Brobnar_RTivg44.png"
},
#...more data was in here but there is URL limit
{
"id": "Mars",
"name": "Mars",
"image": "https://cdn.keyforgegame.com/media/houses/Mars_CmAUCXI.png"
}
],
"cards": [
{
"id": "469dd68d-cdd6-40e0-8fc9-a167c45a9aea",
"card_title": "Grasping Vines",
"house": "Untamed",
"card_type": "Action",
"front_image": "https://cdn.keyforgegame.com/media/card_front/en/341_324_VH9R4P26824V_en.png",
"card_text": "Play: Return up to 3 artifacts to their owners’ hands.",
"traits": null,
"amber": 1,
"power": 0,
"armor": 0,
"rarity": "Uncommon",
"flavor_text": null,
"card_number": 324,
"expansion": 341,
"is_maverick": false
},
{
"id": "1a84631d-7fcb-4c9a-a50c-9539dcb84928",
"card_title": "Yxilo Bolter",
"house": "Mars",
"card_type": "Creature",
"front_image": "https://cdn.keyforgegame.com/media/card_front/en/341_204_H9HQ5F59FJQX_en.png",
"card_text": "Fight/Reap: Deal 2<D> to a creature. If this damage destroys that creature, purge it.",
"traits": "Martian • Soldier",
"amber": 0,
"power": 3,
"armor": 0,
"rarity": "Common",
"flavor_text": null,
"card_number": 204,
"expansion": 341,
"is_maverick": false
}, #the data goes on for quite a bit
import json
import requests
import sqlite3
from datetime import datetime
DATE_FORMAT = '%Y-%m-%d'
class Keyforge():
# def __init__(self):
def download(self,pagestart,pagestop):
#Downloads batches of 25 decks per page. pagestart=1,pagestop=5 will download the first 100 decks in the master vault
for i in range(pagestart,pagestop):
response = requests.get("https://www.keyforgegame.com/api/decks/?page="+str(i)+"&links=cards&page_size=25")
raw_data = json.loads(response.text)
someitem = raw_data.items()
columns = list(someitem)
print (columns)
with open("keyforge_scraped"+str(i)+".json","w") as data_file:
json.dump(raw_data,data_file,indent=2)
print ("Generating batch number %d." % i)
conn = sqlite3.connect('keyforge.db')
c = conn.cursor()
c.execute("CREATE TABLE "keyforge" ( `deck number` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, `deck name` TEXT, `cardID0` INTEGER, `cardID1` INTEGER, `cardID2` INTEGER, `cardID3` INTEGER, `cardID4` INTEGER, `cardID5` INTEGER, `cardID6` INTEGER, `cardID7` INTEGER, `cardID8` INTEGER, `cardID9` INTEGER, `cardID10` INTEGER, `cardID11` INTEGER, `cardID12` INTEGER, `cardID13` INTEGER, `cardID14` INTEGER, `cardID15` INTEGER, `cardID16` INTEGER, `cardID17` INTEGER, `cardID18` INTEGER, `cardID19` INTEGER, `cardID20` INTEGER, `cardID21` INTEGER, `cardID22` INTEGER, `cardID23` INTEGER, `cardID24` INTEGER, `cardID25` INTEGER, `cardID26` INTEGER, `cardID27` INTEGER, `cardID28` INTEGER, `cardID29` INTEGER, `cardID30` INTEGER, `cardID31` INTEGER, `cardID32` INTEGER, `cardID33` INTEGER, `cardID34` INTEGER, `cardID35` INTEGER, `Wins` INTEGER, `Losses` INTEGER, `Casual wins` INTEGER, `Casual losses` INTEGER )")
for i in raw_data:
c.execute("insert into keyforge values (columns[1],?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",columns)
conn.commit()
conn.close()
return raw_data
""" def updateDB(raw_data):
conn = sqlite3.connect('keyforge.db')
c = conn.cursor()
c.execute("CREATE TABLE IF NOT EXISTS keyforge (id varchar(3), raw_data json)")
for i in raw_data:
c.execute("insert into keyforge values (?,?)",[i['id'], json.dumps(raw_data)])
conn.commit()
conn.close()
"""
start = int(input("Enter start page: "))
stop = int(input("Enter stop page: "))
a = Keyforge()
raw_data = a.download(start,stop)
#a.updateDB(raw_data)
通过尝试许多不同的操作，您可以看到我在代码中创建的混乱情况。我已经成功地从网站上下载了JSON对象并将其归档到磁盘上，但是sqlite数据库却没有运气。
我已经使用GUI手动创建了我想要的表，但是我不知道如何从JSON数据中填充此类表。
最终，我要运行我的keyforge.py代码，选择要下载和处理的页面（每个页面包含25组数据），要下载和处理（在磁盘中存档，并在数据库中存储），以及以便随时间更新和增加此数据库。显然，我还将处理数据以将其馈送到我的学习算法中。
你们能帮我一下吗？我被困在洞里。请帮我摆脱困境：）