{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example Usage of TopicGPT: Amazon Reviews"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we will be using the Amazon Reviews dataset to show how TopicGPT can be useful when analyzing a large corpus of text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\umap\\distances.py:1063: NumbaDeprecationWarning: \u001b[1mThe 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\u001b[0m\n",
      "  @numba.jit()\n",
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\umap\\distances.py:1071: NumbaDeprecationWarning: \u001b[1mThe 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\u001b[0m\n",
      "  @numba.jit()\n",
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\umap\\distances.py:1086: NumbaDeprecationWarning: \u001b[1mThe 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\u001b[0m\n",
      "  @numba.jit()\n",
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\umap\\umap_.py:660: NumbaDeprecationWarning: \u001b[1mThe 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\u001b[0m\n",
      "  @numba.jit()\n",
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\umap\\plot.py:203: NumbaDeprecationWarning: \u001b[1mThe 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\u001b[0m\n",
      "  @numba.jit()\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\arik_\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\arik_\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from topicgpt.TopicGPT import TopicGPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load api key\n",
    "import os\n",
    "api_key_openai = os.environ.get('OPENAI_API_KEY')\n",
    "\n",
    "import openai\n",
    "\n",
    "openai.organization = \"org-MOfdTrYSke1pXhlAdLXxwDKx\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data from https://www.kaggle.com/datasets/kritanjalijain/amazon-reviews?resource=download\n",
    "\n",
    "review_data = pd.read_csv(\"../Data/AmazonReviews/amazon_review_polarity_csv/train.csv\", header=None) # only use the first 10k reviews of the train set\n",
    "\n",
    "reviews = list(review_data[2])\n",
    "reviews = reviews[:10000] # only consider the first 10k reviews "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm = TopicGPT(\n",
    "    openai_api_key = api_key_openai,\n",
    "    corpus_instruction= \"The Amazon reviews dataset consists of reviews from amazon. The data span a period of 18 years, including 10000 reviews up to March 2013.\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm.fit(reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm.save_embeddings()  #save the computed embeddings for later use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm.visualize_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Topic 0: Musical genres and characteristics,\n",
       " Topic 1: Sci-fi TV show.,\n",
       " Topic 2: Film Genres,\n",
       " Topic 3: Paranormal phenomena and UFO sightings,\n",
       " Topic 4: Earbuds and Headsets,\n",
       " Topic 5: Book Review Topics,\n",
       " Topic 6: Gluten-free Cookbook,\n",
       " Topic 7: Air Mattresses,\n",
       " Topic 8: Crime and Investigation.,\n",
       " Topic 9: Printer Troubleshooting,\n",
       " Topic 10: Hiking Footwear,\n",
       " Topic 11: Shapewear,\n",
       " Topic 12: Dance Instruction,\n",
       " Topic 13: Parenting and Education,\n",
       " Topic 14: Electronic Gadgets,\n",
       " Topic 15: Video Games,\n",
       " Topic 16: MP3 Player Issues,\n",
       " Topic 17: Camera Accessories,\n",
       " Topic 18: Power Adapters,\n",
       " Topic 19: Product Quality,\n",
       " Topic 20: Ancient civilizations and anthropology.,\n",
       " Topic 21: Router Connectivity,\n",
       " Topic 22: Technical Issues,\n",
       " Topic 23: Puritanical Society,\n",
       " Topic 24: Sci-fi Space Exploration,\n",
       " Topic 25: Beauty Products,\n",
       " Topic 26: Sexual Vibrators,\n",
       " Topic 27: Home Safety,\n",
       " Topic 28: Product Quality,\n",
       " Topic 29: Customer Service Experience,\n",
       " Topic 30: Textbook Quality,\n",
       " Topic 31: Programming Documentation,\n",
       " Topic 32: Hardware Tools,\n",
       " Topic 33: Product Quality,\n",
       " Topic 34: Educational Toys,\n",
       " Topic 35: Appliances,\n",
       " Topic 36: Kitchenware,\n",
       " Topic 37: Supernatural Witches,\n",
       " Topic 38: Horror Comics,\n",
       " Topic 39: Dystopian society,\n",
       " Topic 40: Emotional Turmoil,\n",
       " Topic 41: Book genres,\n",
       " Topic 42: Economic and Political Critique,\n",
       " Topic 43: Poorly Written Erotica,\n",
       " Topic 44: Dystopian Surveillance State,\n",
       " Topic 45: Experimental Poetry,\n",
       " Topic 46: Formatting Issues,\n",
       " Topic 47: Language Learning Resources,\n",
       " Topic 48: Book genres,\n",
       " Topic 49: Home Improvement,\n",
       " Topic 50: Religious Texts.]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tm.topic_lis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\umap\\distances.py:1063: NumbaDeprecationWarning: \u001b[1mThe 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\u001b[0m\n",
      "  @numba.jit()\n",
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\umap\\distances.py:1071: NumbaDeprecationWarning: \u001b[1mThe 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\u001b[0m\n",
      "  @numba.jit()\n",
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\umap\\distances.py:1086: NumbaDeprecationWarning: \u001b[1mThe 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\u001b[0m\n",
      "  @numba.jit()\n",
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\umap\\umap_.py:660: NumbaDeprecationWarning: \u001b[1mThe 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\u001b[0m\n",
      "  @numba.jit()\n",
      "c:\\Users\\arik_\\anaconda3\\envs\\llm_sem_test7\\Lib\\site-packages\\umap\\plot.py:203: NumbaDeprecationWarning: \u001b[1mThe 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\u001b[0m\n",
      "  @numba.jit()\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\arik_\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\arik_\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wed Sep  6 20:17:07 2023 Building and compiling search function\n"
     ]
    }
   ],
   "source": [
    "# load the model if available\n",
    "import pickle\n",
    "with open(\"../Data/SavedTopicRepresentations/TopicGPT_amazonReviews.pkl\", \"rb\") as f:\n",
    "    tm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see what topic 2 is about"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The common topic of the given words is \"Movie Reviews\".\n",
      "\n",
      "Aspects:\n",
      "1. Genre: animated, slasher, noir, zombie, thriller.\n",
      "2. Quality: watchable, unwatchable, dreadful, cheesy, ridiculous.\n",
      "3. Filmmaking: directors, filmmakers, screenwriter, cinematography, filmmaking.\n",
      "4. Audience reaction: scariest, thrilling, hilarious, disappointing, shocking.\n",
      "5. Technical aspects: widescreen, dolby, surround, cinematography, special effects.\n"
     ]
    }
   ],
   "source": [
    "print(tm.topic_lis[2].topic_description)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT wants to the call the function:  {\n",
      "  \"name\": \"identify_topic_idx\",\n",
      "  \"arguments\": \"{\\n  \\\"query\\\": \\\"Avatar\\\"\\n}\"\n",
      "}\n",
      "GPT wants to the call the function:  {\n",
      "  \"name\": \"get_topic_information\",\n",
      "  \"arguments\": \"{\\n  \\\"topic_idx_lis\\\": [2]\\n}\"\n",
      "}\n",
      "Yes, the movie \"Avatar\" is mentioned in topic 2, which is about film genres. However, the specific context or sentiment of the mention is not provided.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{2: '\\n            Topic index: 2\\n            Topic name: Film Genres\\n            Topic description: The common topic of the given words is \"Movie Reviews\".\\n\\nAspects:\\n1. Genre: animated, slasher, noir, zombie, thriller.\\n2. Quality: watchable, unwatchable, dreadful, cheesy, ridiculous.\\n3. Filmmaking: directors, filmmakers, screenwriter, cinematography, filmmaking.\\n4. Audience reaction: scariest, thrilling, hilarious, disappointing, shocking.\\n5. Technical aspects: widescreen, dolby, surround, cinematography, special effects.\\n            Topic topwords: [\\'theaters\\', \\'flicks\\', \\'renting\\', \\'animated\\', \\'robots\\', \\'gang\\', \\'rental\\', \\'filmmakers\\', \\'aerial\\', \\'unrated\\', \\'dubbing\\', \\'credits\\', \\'accents\\', \\'slasher\\', \\'scares\\', \\'noir\\', \\'cop\\', \\'cartoons\\', \\'unwatchable\\', \\'watchable\\', \\'directors\\', \\'porn\\', \\'scariest\\', \\'fights\\', \\'aired\\', \\'adaptation\\', \\'locations\\', \\'wrestling\\', \\'zombie\\', \\'corny\\', \\'watches\\', \\'screenwriter\\', \\'chicks\\', \\'theatrical\\', \\'lesbian\\', \\'sequels\\', \\'villa\\', \\'countryside\\', \\'grabs\\', \\'producer\\', \\'combo\\', \\'redeem\\', \\'trapped\\', \\'marine\\', \\'idiot\\', \\'depicted\\', \\'storylines\\', \\'millions\\', \\'ruin\\', \\'flop\\', \\'thrills\\', \\'claustrophobic\\', \\'caves\\', \\'non-stop\\', \\'blockbuster\\', \\'soldier\\', \\'actresses\\', \\'cinematic\\', \\'breathtaking\\', \\'comedian\\', \\'theatres\\', \\'pilots\\', \\'airplanes\\', \\'widescreen\\', \\'steals\\', \\'grainy\\', \\'acclaimed\\', \\'surround\\', \\'channels\\', \\'innovative\\', \\'buff\\', \\'talents\\', \\'motorcycle\\', \\'soccer\\', \\'courage\\', \\'distracting\\', \\'scenario\\', \\'cult\\', \\'portrays\\', \\'compelled\\', \\'scale\\', \\'segments\\', \\'butt\\', \\'promising\\', \\'cliched\\', \\'sub\\', \\'artsy\\', \\'faces\\', \\'kills\\', \\'rubbish\\', \\'involving\\', \\'vulgar\\', \\'incoherent\\', \\'costumes\\', \\'accent\\', \\'disjointed\\', \\'gory\\', \\'gruesome\\', \\'low-budget\\', \\'dreadful\\', \\'scream\\', \\'host\\', \\'fright\\', \\'action-packed\\', \\'goofy\\', \\'caving\\', \\'driven\\', \\'nude\\', \\'re-make\\', \\'audiences\\', \\'over-the-top\\', \\'filmmaking\\', \\'popcorn\\', \\'aircraft\\', \\'newer\\', \\'sport\\', \\'kicks\\', \\'ventriloquist\\', \\'crying\\', \\'rescue\\', \\'buffs\\', \\'deluxe\\', \\'cheated\\', \\'lackluster\\', \\'dolby\\', \\'strangers\\', \\'pun\\', \\'pitiful\\', \\'adore\\', \\'suffers\\', \\'initially\\', \\'loser\\', \\'arts\\', \\'shocking\\', \\'criminals\\', \\'weapons\\', \\'lion\\', \\'eerie\\', \\'preview\\', \\'youth\\', \\'parent\\', \\'poignant\\', \\'presence\\', \\'decade\\', \\'sympathetic\\', \\'interview\\', \\'achieve\\', \\'jobs\\', \\'motivation\\', \\'blonde\\', \\'poem\\', \\'transitions\\', \\'convincing\\', \\'marketing\\', \\'consumers\\', \\'candy\\', \\'asks\\', \\'walked\\', \\'miscast\\', \\'stunts\\', \\'nasty\\', \\'zombies\\', \\'thrill\\', \\'vomit\\', \\'stupidity\\', \\'stinker\\', \\'freaking\\', \\'disgusting\\', \\'ridiculously\\', \\'farce\\', \\'chills\\', \\'brainless\\', \\'dude\\', \\'naked\\', \\'ratings\\', \\'appalling\\', \\'pretends\\', \\'disgust\\', \\'filming\\', \\'starred\\', \\'punches\\', \\'downhill\\', \\'des\\', \\'sappy\\', \\'breathless\\', \\'screening\\', \\'funniest\\', \\'rude\\', \\'tripe\\', \\'hairy\\', \\'handsome\\', \\'unfunny\\', \\'genuinely\\', \\'fighters\\', \\'spoiled\\', \\'fighter\\', \\'backdrop\\', \\'dogfight\\', \\'planes\\', \\'aviation\\', \\'fast-paced\\', \\'dogfights\\', \\'chasing\\', \\'destruction\\', \\'dramas\\', \\'silent\\', \\'specials\\', \\'westerns\\', \\'permanent\\', \\'blurry\\', \\'blu-rays\\', \\'bikers\\', \\'crazed\\', \\'transfers\\', \\'cue\\', \\'deaf\\', \\'skipping\\', \\'commercials\\', \\'gritty\\', \\'exit\\', \\'jumped\\', \\'continuity\\', \\'robbed\\', \\'knocked\\', \\'sync\\', \\'boxed\\', \\'studios\\', \\'remastered\\', \\'pacing\\', \\'boat\\', \\'headed\\', \\'lonely\\', \\'misfortune\\', \\'marry\\', \\'beg\\', \\'jungle\\', \\'alley\\', \\'suffer\\', \\'victim\\', \\'sympathy\\', \\'wouldnt\\', \\'hurts\\', \\'handled\\', \\'shouting\\', \\'crawl\\', \\'border\\', \\'intent\\', \\'flashbacks\\', \\'viewed\\', \\'spite\\', \\'throat\\', \\'sacrificing\\', \\'ticket\\', \\'painfully\\', \\'passionate\\', \\'martial\\', \\'narration\\', \\'reaction\\', \\'deaths\\', \\'neighbors\\', \\'offend\\', \\'visually\\', \\'perverse\\', \\'realy\\', \\'goal\\', \\'tender\\', \\'portray\\', \\'bullets\\', \\'engaged\\', \\'coverage\\', \\'monotonous\\', \\'unlikely\\', \\'historically\\', \\'banal\\', \\'credibility\\', \\'one-liners\\', \\'depicts\\', \\'caring\\', \\'divorced\\', \\'grave\\', \\'sincere\\', \\'reaches\\', \\'meaningful\\', \\'mild\\', \\'souls\\', \\'downright\\', \\'dramatically\\', \\'involves\\', \\'understatement\\', \\'hates\\', \\'crosses\\', \\'workers\\', \\'interactions\\', \\'overwhelming\\', \\'statues\\', \\'sum\\', \\'photographed\\', \\'ranks\\', \\'aged\\', \\'region\\', \\'post-apocalyptic\\', \\'spoil\\', \\'dud\\', \\'qualities\\', \\'merit\\', \\'borrowed\\', \\'adapted\\', \\'scripts\\', \\'weaker\\', \\'justify\\', \\'purposes\\', \\'coherent\\', \\'gratuitous\\', \\'creature\\', \\'segment\\', \\'whim\\', \\'determine\\', \\'firefighters\\', \\'ok.\\', \\'inaccurate\\', \\'warmth\\', \\'turkey\\', \\'installment\\', \\'picky\\', \\'remotely\\', \\'shell\\', \\'receipt\\', \\'complained\\', \\'phoned\\', \\'cheesey\\', \\'shooting\\', \\'freak\\', \\'cheezy\\', \\'shoots\\', \\'travesty\\', \\'plotless\\', \\'drunk\\', \\'vile\\', \\'spy\\', \\'laughably\\', \\'shameless\\', \\'actors/actresses\\', \\'screams\\', \\'embarrassing\\', \\'rape\\', \\'embarrassed\\', \\'foul\\', \\'half-hour\\', \\'chases\\', \\'retarded\\', \\'crawling\\', \\'spoiler\\', \\'lustful\\', \\'pissed\\', \\'filmmaker\\', \\'disgrace\\', \\'spliced\\', \\'depths\\', \\'uncensored\\', \\'originality\\', \\'twins\\', \\'must-see\\', \\'unreal\\', \\'mansion\\', \\'cameo\\', \\'rendering\\', \\'marvelous\\', \\'comedies\\', \\'crippled\\', \\'comedians\\', \\'marvel\\', \\'fought\\', \\'ghostly\\', \\'spooky\\', \\'bare\\', \\'phenomenal\\', \\'robot\\', \\'underrated\\', \\'beneath\\', \\'comical\\', \\'landing\\', \\'crew\\', \\'heartwarming\\', \\'mute\\', \\'finale\\', \\'teaser\\', \\'airplane\\', \\'nonetheless\\', \\'campy\\', \\'autobots\\', \\'five-star\\', \\'beating\\', \\'reruns\\', \\'marries\\', \\'fond\\', \\'flawless\\', \\'avenger\\', \\'cavalry\\', \\'faded\\', \\'laughter\\', \\'streaming\\', \\'viewings\\', \\'pixelated\\', \\'letterboxed\\', \\'dub\\', \\'biker\\', \\'bluray\\', \\'spiderman\\', \\'beloved\\', \\'beautifull\\', \\'broadcast\\', \\'boxset\\', \\'swinging\\', \\'restored\\', \\'captioning\\', \\'organ\\', \\'skips\\', \\'previews\\', \\'disapointment\\', \\'tickets\\', \\'peanuts\\', \\'holidays\\', \\'insomnia\\', \\'geared\\', \\'suit\\', \\'perfection\\', \\'crisp\\', \\'mesmerizing\\', \\'butter\\', \\'bus\\', \\'vehicle\\', \\'cloying\\', \\'butchered\\', \\'slap\\', \\'spots\\', \\'angles\\', \\'builds\\', \\'tossed\\', \\'facial\\', \\'ripping\\', \\'jumping\\', \\'manufactured\\', \\'glued\\', \\'struck\\', \\'idiots\\', \\'cliff\\', \\'bullet\\', \\'accident\\', \\'performs\\', \\'shakes\\', \\'lossless\\', \\'expired\\', \\'yell\\', \\'grass\\', \\'hurl\\', \\'walks\\', \\'flesh\\', \\'quirky\\', \\'sooo\\', \\'expedition\\', \\'stilted\\', \\'so-so\\', \\'sue\\', \\'cheating\\', \\'regrettably\\', \\'cried\\', \\'alike\\', \\'afterwards\\', \\'escapes\\', \\'headache\\', \\'den\\', \\'shy\\', \\'bisexual\\', \\'effeminate\\', \\'wit\\', \\'suffering\\', \\'cabin\\', \\'minimal\\', \\'riveting\\', \\'brow\\', \\'racism\\', \\'thankfully\\', \\'kinky\\', \\'songwriter\\', \\'mind-numbing\\', \\'leap\\', \\'artifacts\\', \\'attraction\\', \\'enticing\\', \\'mill\\', \\'collectors\\', \\'whiny\\', \\'bickering\\', \\'flawed\\', \\'ton\\', \\'adopted\\', \\'tortured\\', \\'assumed\\', \\'nominated\\', \\'maintain\\']'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tm.pprompt(\"Is the movie Avatar mentioned in topic 2?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To check the output, we actually inspect the respective document at index 1498: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not just because of the 3D, but because this is the version where they made an effort to optimize the picture! They released this on blu ray like like avatar was released. First they release the movie without any optimization and special features. Which means the picture looks better than DVD but not the best that blu ray can be.(which means a grainy looking picture that looks like the characters are in a sandstorm and there is a lack of detail that you expect in a blu ray). The they make the limited edition which is made the way a blu ray is supposed to be down. So if you are wondering which one to choose, this is the one you want! All the features with the visuals to boot!\n"
     ]
    }
   ],
   "source": [
    "print(tm.topic_lis[2].documents[1498])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us go own with the analysis. Since it is easy to loose the overview over all the topics, lets find out which one is about books"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT wants to the call the function:  {\n",
      "  \"name\": \"identify_topic_idx\",\n",
      "  \"arguments\": \"{\\n  \\\"query\\\": \\\"books\\\"\\n}\"\n",
      "}\n",
      "Topic 5 is about books.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tm.pprompt(\"Which topic is about books?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The common topic of the given words is \"Book Reviews\". \n",
      "\n",
      "Various aspects and sub-topics of this topic include:\n",
      "1. Characters: likable, endearing, mighty, crew, pals\n",
      "2. Storyline: satirical, mythological, strange, satirical, endings\n",
      "3. Writing style: well-crafted, inviting, brilliantly, sarcastic\n",
      "4. Themes: philosophical, allusions, belief, religion, obsession\n",
      "5. Critique: uneven, dissatisfaction, novice, pale, unsuccessful\n"
     ]
    }
   ],
   "source": [
    "print(tm.topic_lis[5].topic_description)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT wants to the call the function:  {\n",
      "  \"name\": \"knn_search\",\n",
      "  \"arguments\": \"{\\n  \\\"topic_index\\\": 4,\\n  \\\"query\\\": \\\"Harry Potter\\\",\\n  \\\"k\\\": 5\\n}\"\n",
      "}\n",
      "No, there is no mention of Harry Potter in topic 5. The documents that are most closely related to the query \"Harry Potter\" do not mention the topic.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([\"Unable to use. Compartments too tiny and too deep to reach in to get earrings - and I don't have unusually large fingers.\",\n",
       "  \"I wanted an in ear blue tooth headset but couldn't get them to stay in. these made it work!\",\n",
       "  'people should buy headsets to fit these things bc they are so essential..A must have for any headset that will fit them',\n",
       "  'I am am a musician so I use these with earbuds coming off my computer within my headphones which is powered from my Marshall Amplifier. They transfer the sound well and are very well made.',\n",
       "  'The Jabra Eargels are wonderful.. I cannot believe how comfortable, and user friendly they are. Thank you, if not for these, I could not use my bluetooth.Great Merchandise..'],\n",
       " [71, 27, 30, 72, 69])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tm.pprompt(\"Is Harry Potter mentioned in topic 5?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Topic 0 (\"Musical genres and characteristics\") sounds a bit general and from the visual inspection it seems to contain a lot of documents. So let's break it down a little bit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT wants to the call the function:  {\n",
      "  \"name\": \"split_topic_kmeans\",\n",
      "  \"arguments\": \"{\\n  \\\"topic_idx\\\": 0,\\n  \\\"inplace\\\": true\\n}\"\n",
      "}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epochs completed: 100%| ██████████ 100/100 [00:03]\n",
      "Computing word-topic matrix: 100%|██████████| 1/1 [00:01<00:00,  1.29s/it]\n",
      "Epochs completed: 100%| ██████████ 100/100 [00:01]\n",
      "Epochs completed: 100%| ██████████ 100/100 [00:01]\n",
      "  0%|          | 0/1 [00:00<?, ?it/s]"
     ]
    }
   ],
   "source": [
    "tm.pprompt(\"please split topic 0 into subtopics. Do this inplace\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that the topic 0 was split into two topics. One on music genres and the other one on remastering of music. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "On the other hand, topic 0 and topic 2 seem very similar. Let's find out more about the difference between the two: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT wants to the call the function:  {\n",
      "  \"name\": \"get_topic_information\",\n",
      "  \"arguments\": \"{\\n  \\\"topic_idx_lis\\\": [0, 2]\\n}\"\n",
      "}\n",
      "Topic 0 is about sci-fi TV shows, while topic 2 is about paranormal phenomena and UFO sightings. \n",
      "\n",
      "The similarities between the two topics seem to be quite limited based on the available descriptions. Both topics involve speculative elements and may touch on the realm of the unknown. However, the specific sub-topics and aspects mentioned in the descriptions of the topics do not share much overlap.\n",
      "\n",
      "Topic 0, the sci-fi TV show topic, includes sub-topics such as plot and story, setting and characters, originality and creativity, quality and highlights, and viewer experience and opinions. The top words associated with this topic include terms like \"finale\", \"timeline\", \"rewind\", \"hints\", \"galaxy\", \"starship\", \"crew\", and \"originality\".\n",
      "\n",
      "Topic 2, the paranormal and UFO sightings topic, includes sub-topics and aspects such as UFO sightings, unexplained events, belief and speculation, controversy and debunking, and involvement and research. The top words associated with this topic include terms like \"sightings\", \"paranormal\", \"phenomena\", \"UFO\", \"encounters\", \"unexplained\", \"speculation\", and \"believer\".\n",
      "\n",
      "In summary, while both topics may involve elements of speculation and the unknown, they seem to focus on different subject matter – sci-fi TV shows for topic 0 and paranormal phenomena and UFO sightings for topic 2.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{0: '\\n            Topic index: 0\\n            Topic name: Sci-fi TV show.\\n            Topic description: The common topic of the given words is \"TV shows\" or \"television series\". \\n\\nSub-topics and aspects of this topic include:\\n1. Plot and story: \"finale\", \"timeline\", \"rewind\", \"rehash\", \"hints\"\\n2. Setting and characters: \"galaxy\", \"starship\", \"crew\", \"diehard\"\\n3. Originality and creativity: \"originality\", \"spawned\", \"flawless\"\\n4. Quality and highlights: \"highlights\", \"vice\", \"pacing\", \"woefully\", \"kinks\"\\n5. Viewer experience and opinions: \"farewell\", \"so-so\", \"unintentionally\", \"desperate\", \"on-going\"\\n            Topic topwords: [\\'finale\\', \\'timeline\\', \\'gate\\', \\'rewind\\', \\'rehash\\', \\'hints\\', \\'galaxy\\', \\'served\\', \\'continuation\\', \\'crew\\', \\'starship\\', \\'diehard\\', \\'originality\\', \\'aired\\', \\'orginal\\', \\'spawned\\', \\'flawless\\', \\'highlights\\', \\'vice\\', \\'pacing\\', \\'woefully\\', \\'kinks\\', \\'farewell\\', \\'so-so\\', \\'unintentionally\\', \\'desperate\\', \\'on-going\\', \\'watcher\\', \\'verry\\', \\'crimes\\', \\'arctic\\', \\'wicked\\', \\'stance\\', \\'swearing\\', \\'remaining\\', \\'namely\\', \\'settled\\', \\'tackle\\', \\'travelling\\', \\'babble\\', \\'trial\\', \\'appearances\\', \\'self-absorbed\\', \\'tricks\\', \\'peek\\', \\'eager\\', \\'guard\\', \\'gould\\', \\'span\\', \\'dedicated\\', \\'tying\\', \\'saga\\', \\'intentionally\\', \\'convoluted\\', \\'zing\\', \\'truely\\', \\'devoid\\', \\'dependent\\', \\'infinite\\', \\'parade\\', \\'loyal\\', \\'altered\\', \\'poignant\\', \\'parallel\\', \\'branch\\', \\'profoundly\\', \\'committed\\', \\'resolve\\', \\'gaps\\', \\'trek\\', \\'judgement\\', \\'philosophical\\', \\'entry\\', \\'souls\\', \\'sci\\', \\'versa\\', \\'crammed\\', \\'rocked\\', \\'integrated\\', \\'revisit\\', \\'manipulate\\', \\'owning\\', \\'comeback\\', \\'succesful\\', \\'regulations\\', \\'purposes\\', \\'assuming\\', \\'preparing\\', \\'welcome\\', \\'formed\\', \\'fundamental\\', \\'secured\\', \\'concentrated\\', \\'o.k\\', \\'dinner\\', \\'arriving\\', \\'pleasent\\', \\'prefered\\', \\'satisfaction\\', \\'wrap\\', \\'snacks\\', \\'closure\\', \\'potty\\', \\'installment\\', \\'rainy\\', \\'fluff\\', \\'slipped\\', \\'interplay\\', \\'compression\\', \\'prayer\\', \\'pre-historical\\', \\'positions\\', \\'pre\\', \\'positioned\\', \\'positively\\', \\'praying\\', \\'poses\\', \\'precise\\', \\'ports\\', \\'pose\\', \\'portrays\\', \\'portraying\\', \\'preach\\', \\'preacher\\', \\'portrayals\\', \\'portray\\', \\'preaches\\', \\'preaching\\', \\'preachy\\', \\'preceding\\', \\'portrait\\', \\'precious\\', \\'pre-pregnancy\\', \\'possess\\', \\'post-apocalyptic\\', \\'pray\\', \\'pots\\', \\'practiced\\', \\'potter\\', \\'ppl\\', \\'powershot\\', \\'pound\\', \\'powers\\', \\'pounding\\', \\'pounds\\', \\'powerfully\\', \\'powered\\', \\'powder\\', \\'pour\\', \\'poured\\', \\'poverty\\', \\'potentially\\', \\'potente\\', \\'posture\\', \\'posts\\', \\'praising\\', \\'possibilities\\', \\'possibility\\', \\'pow\\', \\'postage\\', \\'postcard\\', \\'praised\\', \\'possesses\\', \\'prague\\', \\'posted\\', \\'poster\\', \\'posters\\', \\'practicing\\', \\'postings\\', \\'practices\\', \\'postpartum\\', \\'praetorians\\', \\'precisely\\', \\'a/c\\', \\'predator\\', \\'profile\\', \\'proficient\\', \\'professors\\', \\'professionals\\', \\'professionally\\', \\'prof.\\', \\'producto\\', \\'productive\\', \\'productions\\', \\'product.the\\', \\'producing\\', \\'produces\\', \\'producer\\', \\'prodigal\\', \\'proclaimed\\', \\'processes\\', \\'procedures\\', \\'procedure\\', \\'problematic\\', \\'probe\\', \\'prob\\', \\'privilege\\', \\'privacy\\', \\'profit\\', \\'profits\\', \\'programmed\\', \\'programmer\\', \\'propaganda\\', \\'prop\\', \\'proofread\\', \\'pronunciation\\', \\'pronounciation\\', \\'pronounce\\', \\'prongs\\', \\'prone\\', \\'prompted\\', \\'prompt\\', \\'promotion\\', \\'pristine\\', \\'promoting\\', \\'promote\\', \\'promo\\', \\'promising\\', \\'promises\\', \\'prominent\\', \\'prologue\\', \\'proles\\', \\'progressive\\', \\'progresses\\', \\'progressed\\', \\'programmers\\', \\'promotes\\', \\'prisoners\\', \\'prison\\', \\'priority\\', \\'presenting\\', \\'presence\\', \\'prescription\\', \\'prescient\\', \\'preparation\\', \\'prep\\', \\'portions\\', \\'premium\\', \\'premiere\\', \\'prelude\\', \\'prejudice\\', \\'preserve\\', \\'pregnancy\\', \\'preference\\', \\'preface\\', \\'predicts\\', \\'predictions\\', \\'prediction\\', \\'predicting\\', \\'predicted\\', \\'predictability\\', \\'predict\\', \\'predecessors\\', \\'predators\\', \\'prefers\\', \\'precision\\', \\'preserved\\', \\'president\\', \\'principles\\', \\'principle\\', \\'principal\\', \\'princess\\', \\'prince\\', \\'primer\\', \\'primal\\', \\'pride\\', \\'pricey\\', \\'priceless\\', \\'prey\\', \\'preset\\', \\'previews\\', \\'preview\\', \\'prevents\\', \\'prevalent\\', \\'pretension\\', \\'pretends\\', \\'pretend\\', \\'presto\\', \\'prestigious\\', \\'pressured\\', \\'pressing\\', \\'pressed\\', \\'previewed\\', \\'portfolio\\', \\'polidoro\\', \\'port\\', \\'ph.d.\\', \\'peyton\\', \\'pets\\', \\'petite\\', \\'peterson\\', \\'peters\\', \\'pete\\', \\'pesky\\', \\'perverse\\', \\'pervasive\\', \\'persuasion\\', \\'phantom\\', \\'perspectives\\', \\'personaly\\', \\'personalities\\', \\'persona\\', \\'perry\\', \\'perpetual\\', \\'perpetrated\\', \\'pero\\', \\'permanently\\', \\'permanent\\', \\'perks\\', \\'perkins\\', \\'persons\\', \\'periods\\', \\'phase\\', \\'phd\\', \\'pianist\\', \\'piaa\\', \\'physics\\', \\'physically\\', \\'phrasing\\', \\'phrasebook\\', \\'phrase\\', \\'photosmart\\', \\'photographs\\', \\'photographer\\', \\'photographed\\', \\'phases\\', \\'phony\\', \\'phoned\\', \\'phoenix\\', \\'philosophies\\', \\'philosopher\\', \\'philly\\', \\'phillips\\', \\'philadelphia\\', \\'phil\\', \\'phenomenon\\', \\'phenomenal\\', \\'phenomena\\', \\'phones\\', \\'performs\\', \\'performers\\', \\'performer\\', \\'peak\\', \\'peaceful\\', \\'pays\\', \\'payed\\', \\'pause\\', \\'patti\\', \\'pattern\\', \\'patrol\\', \\'patinkin\\', \\'patients\\', \\'patient\\', \\'peaks\\', \\'paths\\', \\'patches\\', \\'patched\\', \\'patch\\', \\'pat\\', \\'pastor\\', \\'paste\\', \\'password\\', \\'passionately\\', \\'passionate\\', \\'passes\\', \\'passage\\', \\'pathos\\', \\'peanuts\\', \\'peck\\', \\'pedal\\', \\'perfection\\', \\'percussion\\', \\'perceptions\\', \\'perception\\', \\'percent\\', \\'perabo\\', \\'peppered\\', \\'pentium\\', \\'pens\\', \\'pennsylvania\\', \\'penguin\\', \\'penetrating\\', \\'penelope\\', \\'pencil\\', \\'penchant\\', \\'pen\\', \\'pellegrino\\', \\'peice\\', \\'pegs\\', \\'peggy\\', \\'peers\\', \\'peeling\\', \\'peel\\', \\'pedestrian\\', \\'pedantic\\', \\'pic\\', \\'portability\\', \\'pickup\\', \\'pics\\', \\'poison\\', \\'pointlessly\\', \\'pointing\\', \\'pointed\\', \\'poets\\', \\'poetic\\', \\'poet\\', \\'poems\\', \\'poem\\', \\'poe\\', \\'podcast\\', \\'poisoning\\', \\'pockets\\', \\'plugs\\', \\'plugging\\', \\'plug-in\\', \\'plotting\\', \\'plotted\\', \\'plotline\\', \\'plotless\\', \\'plot.it\\', \\'plodding\\', \\'plight\\', \\'pliers\\', \\'pny\\', \\'plethora\\', \\'poking\\', \\'polemic\\', \\'porn\\', \\'por\\', \\'population\\', \\'popularity\\', \\'populace\\', \\'pops\\', \\'popping\\', \\'popcorn\\', \\'pop-up\\', \\'poorest\\', \\'poop\\', \\'pole\\', \\'pool\\', \\'ponderous\\', \\'pondering\\', \\'ponder\\', \\'pompous\\', \\'politicians\\', \\'politically\\', \\'polishes\\', \\'polish\\', \\'properties\\', \\'policies\\', \\'policeman\\', \\'pooch\\', \\'plentiful\\', \\'pledge\\', \\'pleasures\\', \\'pitched\\', \\'pitch\\', \\'pita\\', \\'pit\\', \\'pistols\\', \\'pissed\\', \\'piper\\', \\'pipe\\', \\'pioneers\\', \\'pioneer\\', \\'pins\\', \\'pitchshifter\\', \\'pinnacle\\', \\'pinch\\', \\'pilots\\', \\'pilot\\', \\'pills\\', \\'pillows\\', \\'pill\\', \\'pilgrimage\\', \\'pig\\', \\'pierre\\', \\'pieced\\', \\'pie\\', \\'pine\\', \\'pitfalls\\', \\'pitiful\\', \\'pivotal\\', \\'pleasurable\\', \\'pleasing\\', \\'pleasantly\\', \\'playstation\\', \\'playset\\', \\'playlists\\', \\'playlist\\', \\'playboy\\', \\'playback\\', \\'playable\\', \\'plausible\\', \\'platform\\', \\'plates\\', \\'plate\\', \\'plants\\', \\'plant\\', \\'plans\\']',\n",
       " 2: '\\n            Topic index: 2\\n            Topic name: Paranormal phenomena and UFO sightings\\n            Topic description: The common topic of the given words is \"Paranormal Phenomena\". \\n\\nAspects and sub-topics of the topic include:\\n1. UFO Sightings: Encounters, Witnesses, Reported, Sightings\\n2. Unexplained Events: Unexplained, Anecdotes, Incident, Occurrences\\n3. Belief and Speculation: Believer, Speculation, Believed, Possibilities\\n4. Controversy and Debunking: Controversial, Debunked, Dubious, Tendency\\n5. Involvement and Research: Involvement, Researching, Observations, Findings\\n            Topic topwords: [\\'sightings\\', \\'paranormal\\', \\'phenomena\\', \\'ufo\\', \\'encounters\\', \\'unexplained\\', \\'b/c\\', \\'speculation\\', \\'believer\\', \\'relates\\', \\'anecdotes\\', \\'copyright\\', \\'sorely\\', \\'disjointed\\', \\'buff\\', \\'floating\\', \\'visited\\', \\'witnesses\\', \\'stormy\\', \\'admits\\', \\'bin\\', \\'rarity\\', \\'reported\\', \\'ties\\', \\'sighting\\', \\'entitled\\', \\'throws\\', \\'memoir\\', \\'wins\\', \\'stance\\', \\'letdown\\', \\'journalist\\', \\'surrounding\\', \\'hunting\\', \\'incident\\', \\'scenario\\', \\'credibility\\', \\'preview\\', \\'happenings\\', \\'admission\\', \\'reports\\', \\'perpetrated\\', \\'nut\\', \\'delve\\', \\'aunt\\', \\'rampant\\', \\'believed\\', \\'capable\\', \\'touches\\', \\'therein\\', \\'insulting\\', \\'lunatics\\', \\'located\\', \\'pedestrian\\', \\'glance\\', \\'controversial\\', \\'participants\\', \\'conspiracy\\', \\'notably\\', \\'trait\\', \\'possibilities\\', \\'loosely\\', \\'observations\\', \\'involvement\\', \\'hypothesis\\', \\'dubious\\', \\'findings\\', \\'tendency\\', \\'debunked\\', \\'accounts\\', \\'affect\\', \\'positions\\', \\'skeptical\\', \\'pushes\\', \\'mildly\\', \\'sap\\', \\'obsession\\', \\'reporting\\', \\'woo\\', \\'engage\\', \\'assure\\', \\'informs\\', \\'distracted\\', \\'goddesses\\', \\'activity\\', \\'sincere\\', \\'underlying\\', \\'researching\\', \\'applied\\', \\'forgetting\\', \\'behaving\\', \\'theoretical\\', \\'origins\\', \\'occurrences\\', \\'origin\\', \\'plague\\', \\'parallel\\', \\'questionable\\', \\'catagory\\', \\'presenting\\', \\'exhausting\\', \\'generate\\', \\'appearing\\', \\'ramblings\\', \\'mentions\\', \\'documents\\', \\'intrigued\\', \\'unfamiliar\\', \\'approached\\', \\'writings\\', \\'disorganized\\', \\'rang\\', \\'logical\\', \\'concentrated\\', \\'incoherent\\', \\'dross\\', \\'examine\\', \\'understandable\\', \\'thesis\\', \\'adjective\\', \\'teller\\', \\'lengthy\\', \\'blended\\', \\'warns\\', \\'skewed\\', \\'answered\\', \\'reprint\\', \\'scraps\\', \\'storytelling\\', \\'mixture\\', \\'mighty\\', \\'wacky\\', \\'ounce\\', \\'screenwriter\\', \\'goofy\\', \\'wave\\', \\'offbeat\\', \\'phenomenal\\', \\'extraordinary\\', \\'chasing\\', \\'staring\\', \\'aliens\\', \\'dates\\', \\'shocker\\', \\'valid\\', \\'satisfy\\', \\'vehicles\\', \\'ghosts\\', \\'eyed\\', \\'spooky\\', \\'shaped\\', \\'bridge\\', \\'grain\\', \\'gospel\\', \\'composed\\', \\'dose\\', \\'mixes\\', \\'shrink\\', \\'walked\\', \\'ceiling\\', \\'flashlight\\', \\'telephone\\', \\'proles\\', \\'prague\\', \\'programmers\\', \\'possess\\', \\'preach\\', \\'praised\\', \\'praising\\', \\'pray\\', \\'promoting\\', \\'prayer\\', \\'programmer\\', \\'post-apocalyptic\\', \\'praying\\', \\'promotion\\', \\'progresses\\', \\'prompt\\', \\'praetorians\\', \\'postcard\\', \\'pre\\', \\'pre-historical\\', \\'possesses\\', \\'progressed\\', \\'postage\\', \\'prompted\\', \\'prone\\', \\'pre-pregnancy\\', \\'possibility\\', \\'progressive\\', \\'prominent\\', \\'practicing\\', \\'postpartum\\', \\'posts\\', \\'pounds\\', \\'pounding\\', \\'promote\\', \\'pound\\', \\'pour\\', \\'promo\\', \\'posture\\', \\'potty\\', \\'potente\\', \\'potter\\', \\'pots\\', \\'potentially\\', \\'promising\\', \\'prologue\\', \\'postings\\', \\'poverty\\', \\'practices\\', \\'posted\\', \\'practiced\\', \\'ppl\\', \\'powershot\\', \\'powers\\', \\'poured\\', \\'promotes\\', \\'poster\\', \\'promises\\', \\'posters\\', \\'powered\\', \\'powder\\', \\'pow\\', \\'powerfully\\', \\'prisoners\\', \\'profile\\', \\'programmed\\', \\'pretension\\', \\'pretends\\', \\'procedures\\', \\'pretend\\', \\'presto\\', \\'prestigious\\', \\'pressured\\', \\'pressing\\', \\'processes\\', \\'pressed\\', \\'president\\', \\'proclaimed\\', \\'preset\\', \\'preserved\\', \\'prodigal\\', \\'producer\\', \\'preserve\\', \\'produces\\', \\'presence\\', \\'prevalent\\', \\'prescription\\', \\'prevents\\', \\'problematic\\', \\'priority\\', \\'pristine\\', \\'principles\\', \\'principle\\', \\'privacy\\', \\'privilege\\', \\'principal\\', \\'princess\\', \\'prince\\', \\'primer\\', \\'primal\\', \\'pride\\', \\'pricey\\', \\'priceless\\', \\'prob\\', \\'prey\\', \\'previews\\', \\'previewed\\', \\'probe\\', \\'procedure\\', \\'preacher\\', \\'prescient\\', \\'producing\\', \\'predicted\\', \\'predictability\\', \\'predict\\', \\'proficient\\', \\'predecessors\\', \\'predators\\', \\'predator\\', \\'precision\\', \\'precisely\\', \\'precise\\', \\'prison\\', \\'profit\\', \\'profits\\', \\'precious\\', \\'preceding\\', \\'profoundly\\', \\'preachy\\', \\'preaching\\', \\'preaches\\', \\'professors\\', \\'preparing\\', \\'predicting\\', \\'professionals\\', \\'preparation\\', \\'prep\\', \\'premium\\', \\'premiere\\', \\'prelude\\', \\'prejudice\\', \\'pregnancy\\', \\'prefers\\', \\'product.the\\', \\'preference\\', \\'prefered\\', \\'productions\\', \\'productive\\', \\'producto\\', \\'preface\\', \\'predicts\\', \\'predictions\\', \\'prof.\\', \\'professionally\\', \\'prediction\\', \\'a/c\\', \\'pompous\\', \\'positioned\\', \\'philosophies\\', \\'philosophical\\', \\'philosopher\\', \\'philly\\', \\'phillips\\', \\'philadelphia\\', \\'phil\\', \\'phenomenon\\', \\'phd\\', \\'phases\\', \\'phase\\', \\'phoenix\\', \\'phantom\\', \\'peyton\\', \\'pets\\', \\'petite\\', \\'peterson\\', \\'peters\\', \\'pete\\', \\'pesky\\', \\'perverse\\', \\'pervasive\\', \\'persuasion\\', \\'perspectives\\', \\'ph.d.\\', \\'persons\\', \\'phoned\\', \\'phony\\', \\'pilot\\', \\'pills\\', \\'pillows\\', \\'pill\\', \\'pilgrimage\\', \\'pig\\', \\'pierre\\', \\'pieced\\', \\'pie\\', \\'pics\\', \\'picky\\', \\'phones\\', \\'pickup\\', \\'pianist\\', \\'piaa\\', \\'physics\\', \\'physically\\', \\'phrasing\\', \\'phrasebook\\', \\'phrase\\', \\'photosmart\\', \\'photographs\\', \\'photographer\\', \\'photographed\\', \\'pic\\', \\'pilots\\', \\'personaly\\', \\'persona\\', \\'peice\\', \\'pegs\\', \\'peggy\\', \\'peers\\', \\'peeling\\', \\'peel\\', \\'peek\\', \\'pedantic\\', \\'pedal\\', \\'peck\\', \\'peanuts\\', \\'pellegrino\\', \\'peaks\\', \\'peaceful\\', \\'pays\\', \\'payed\\', \\'pause\\', \\'patti\\', \\'pattern\\', \\'patrol\\', \\'patinkin\\', \\'patients\\', \\'patient\\', \\'paths\\', \\'peak\\', \\'personalities\\', \\'pen\\', \\'pencil\\', \\'perry\\', \\'perpetual\\', \\'pero\\', \\'permanently\\', \\'permanent\\', \\'perks\\', \\'perkins\\', \\'periods\\', \\'performs\\', \\'performers\\', \\'performer\\', \\'penchant\\', \\'perfection\\', \\'perceptions\\', \\'perception\\', \\'percent\\', \\'perabo\\', \\'peppered\\', \\'pentium\\', \\'pens\\', \\'pennsylvania\\', \\'penguin\\', \\'penetrating\\', \\'penelope\\', \\'percussion\\', \\'pinch\\', \\'pine\\', \\'pinnacle\\', \\'prongs\\', \\'politicians\\', \\'politically\\', \\'polishes\\', \\'polish\\', \\'polidoro\\', \\'policies\\', \\'policeman\\', \\'polemic\\', \\'pole\\', \\'poking\\', \\'ponder\\', \\'poisoning\\', \\'pointlessly\\', \\'pointing\\', \\'pointed\\', \\'poignant\\', \\'poets\\', \\'poetic\\', \\'poet\\', \\'poems\\', \\'poem\\', \\'poe\\', \\'podcast\\', \\'poison\\', \\'pockets\\', \\'pondering\\', \\'pooch\\', \\'poses\\', \\'pose\\', \\'ports\\', \\'portrays\\', \\'portraying\\', \\'portrayals\\', \\'portray\\', \\'portrait\\', \\'portions\\', \\'portfolio\\', \\'portability\\', \\'ponderous\\', \\'port\\', \\'por\\', \\'population\\', \\'popularity\\', \\'populace\\', \\'pops\\', \\'popping\\', \\'popcorn\\', \\'pop-up\\', \\'poorest\\', \\'poop\\', \\'pool\\', \\'porn\\', \\'pny\\', \\'plugs\\', \\'plugging\\', \\'plant\\', \\'plans\\', \\'planned\\', \\'planets\\', \\'planes\\', \\'plainly\\', \\'placing\\', \\'placement\\', \\'pixelated\\', \\'pivotal\\', \\'pitiful\\', \\'plants\\']'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tm.pprompt(\"What are the differences and similarities of topic 0 and topic 2?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The topics seem fairly related, so we can merge them into one topic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT wants to the call the function:  {\n",
      "  \"name\": \"combine_topics\",\n",
      "  \"arguments\": \"{\\n  \\\"topic_idx_lis\\\": [0, 2],\\n  \\\"inplace\\\": true\\n}\"\n",
      "}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epochs completed: 100%| ██████████ 100/100 [00:02]\n",
      "Computing word-topic matrix: 100%|██████████| 1/1 [00:02<00:00,  2.96s/it]\n",
      "Epochs completed: 100%| ██████████ 30/30 [00:02]\n",
      "Epochs completed: 100%| ██████████ 100/100 [00:01]\n",
      "100%|██████████| 1/1 [00:13<00:00, 13.03s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I have merged topic 0 and topic 2 into a new topic with index 49. The new topic is named \"Movie Genres\".\n",
      "\n",
      "The common topic of the merged topics is \"Entertainment\". Some aspects and sub-topics of this topic include:\n",
      "\n",
      "1. Movies and Theaters: words such as \"flicks\", \"theaters\", \"renting\", \"rental\", \"animated\", \"gang\", \"robots\", and \"producer\" are related to movies and theaters.\n",
      "2. Music and Instruments: words such as \"instruments\", \"acoustic\", \"filmmakers\", \"ballads\", and \"drummer\" are related to music and musical instruments.\n",
      "3. Ratings and Dubbing: words such as \"unrated\", \"dubbing\", \"remix\", \"organ\", and \"acclaimed\" are related to ratings and dubbing in the entertainment industry.\n",
      "4. Genres and Styles: words such as \"innovative\", \"accents\", \"remastered\", \"talents\", and \"instrumental\" are related to different genres and styles of entertainment.\n",
      "5. Thrill and Horror: words such as \"noir\", \"thrill\", \"brutal\", \"unwatchable\", and \"chills\" are related to thrilling and horror-based entertainment.\n",
      "\n",
      "Please note that these descriptions are based on the words you provided and may not cover all possible sub-topics.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[Topic 0: Sci-fi TV show.,\n",
       " Topic 1: Paranormal phenomena and UFO sightings,\n",
       " Topic 2: Earbuds and Headsets,\n",
       " Topic 3: Book Review Topics,\n",
       " Topic 4: Gluten-free Cookbook,\n",
       " Topic 5: Air Mattresses,\n",
       " Topic 6: Crime and Investigation.,\n",
       " Topic 7: Printer Troubleshooting,\n",
       " Topic 8: Hiking Footwear,\n",
       " Topic 9: Shapewear,\n",
       " Topic 10: Dance Instruction,\n",
       " Topic 11: Parenting and Education,\n",
       " Topic 12: Electronic Gadgets,\n",
       " Topic 13: Video Games,\n",
       " Topic 14: MP3 Player Issues,\n",
       " Topic 15: Camera Accessories,\n",
       " Topic 16: Power Adapters,\n",
       " Topic 17: Product Quality,\n",
       " Topic 18: Ancient civilizations and anthropology.,\n",
       " Topic 19: Router Connectivity,\n",
       " Topic 20: Technical Issues,\n",
       " Topic 21: Puritanical Society,\n",
       " Topic 22: Sci-fi Space Exploration,\n",
       " Topic 23: Beauty Products,\n",
       " Topic 24: Sexual Vibrators,\n",
       " Topic 25: Home Safety,\n",
       " Topic 26: Product Quality,\n",
       " Topic 27: Customer Service Experience,\n",
       " Topic 28: Textbook Quality,\n",
       " Topic 29: Programming Documentation,\n",
       " Topic 30: Hardware Tools,\n",
       " Topic 31: Product Quality,\n",
       " Topic 32: Educational Toys,\n",
       " Topic 33: Appliances,\n",
       " Topic 34: Kitchenware,\n",
       " Topic 35: Supernatural Witches,\n",
       " Topic 36: Horror Comics,\n",
       " Topic 37: Dystopian society,\n",
       " Topic 38: Emotional Turmoil,\n",
       " Topic 39: Book genres,\n",
       " Topic 40: Economic and Political Critique,\n",
       " Topic 41: Poorly Written Erotica,\n",
       " Topic 42: Dystopian Surveillance State,\n",
       " Topic 43: Experimental Poetry,\n",
       " Topic 44: Formatting Issues,\n",
       " Topic 45: Language Learning Resources,\n",
       " Topic 46: Book genres,\n",
       " Topic 47: Home Improvement,\n",
       " Topic 48: Religious Texts.,\n",
       " Topic 49: Movie Genres]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tm.pprompt(\"please merge topic 0 and topic 2. Do this inplace\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Topics 39 and 44 seem very similar, so let's find out their difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT wants to the call the function:  {\n",
      "  \"name\": \"get_topic_information\",\n",
      "  \"arguments\": \"{\\n  \\\"topic_idx_lis\\\": [39, 44]\\n}\"\n",
      "}\n",
      "Topic 39 is about \"Firefighters and Dystopian Society.\" It includes sub-topics such as censorship, firefighters, government control, predictions and prophecy, and societal issues. Some related words in this topic are burns, firefighter, outlawed, censored, prohibited, predictions, prophecy, rebellion, and revolution.\n",
      "\n",
      "Topic 44 is about \"Totalitarianism and Surveillance.\" It includes sub-topics such as totalitarianism and oppression, loss of privacy and individuality, grim and nightmarish society, satire and critique of political systems, and predictions and relevance to current society. Some related words in this topic are governments, satire, privacy, surveillance, dystopian, prediction, ruling, propaganda, freedoms, oppression, and liberty.\n",
      "\n",
      "Both topics have dystopian themes and discuss societal issues related to government control and loss of individual privacy. While topic 39 focuses on the role of firefighters in a dystopian society, topic 44 explores the concept of totalitarianism and the impact of surveillance on individual freedom.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{39: '\\n            Topic index: 39\\n            Topic name: Firefighters and Dystopian Society\\n            Topic description: The common topic of the given words is \"dystopian society\". \\n\\nVarious aspects and sub-topics of this topic include:\\n1. Censorship: banned, outlawed, censored, prohibited.\\n2. Firefighters: burns, firefighter, pit, fighter, firefighter.\\n3. Government control: controlling, conformity, enforcement, authority, laws.\\n4. Predictions and prophecy: predictions, prophecy, prophetic, predicts, vision.\\n5. Societal issues: societal, dangers, rebellion, individuality, revolution.\\n            Topic topwords: [\\'burns\\', \\'firefighter\\', \\'houses\\', \\'outlawed\\', \\'metaphor\\', \\'homes\\', \\'science-fiction\\', \\'pat\\', \\'chilling\\', \\'pit\\', \\'fighter\\', \\'burnt\\', \\'banned\\', \\'ironic\\', \\'illegal\\', \\'ignored\\', \\'realizes\\', \\'vain\\', \\'screens\\', \\'censor\\', \\'firefighters\\', \\'predictions\\', \\'dystopian\\', \\'prediction\\', \\'prophecy\\', \\'societal\\', \\'contemplate\\', \\'thought-provoking\\', \\'controlling\\', \\'must-read\\', \\'conformity\\', \\'utopia\\', \\'worlds\\', \\'exposes\\', \\'allusion\\', \\'dangers\\', \\'rebel\\', \\'underlying\\', \\'seventh\\', \\'downright\\', \\'presence\\', \\'cautionary\\', \\'differently\\', \\'scifi\\', \\'epidemic\\', \\'similarities\\', \\'forbidden\\', \\'represented\\', \\'truely\\', \\'pondering\\', \\'experiencing\\', \\'fantastically\\', \\'ironically\\', \\'fruition\\', \\'elaborate\\', \\'interview\\', \\'factors\\', \\'ignorance\\', \\'believed\\', \\'offensive\\', \\'carried\\', \\'schoolers\\', \\'abrupt\\', \\'divided\\', \\'stopping\\', \\'begun\\', \\'beatty\\', \\'demonstrates\\', \\'beg\\', \\'exposed\\', \\'afterward\\', \\'despise\\', \\'inclined\\', \\'fist\\', \\'closing\\', \\'suit\\', \\'offended\\', \\'ratings\\', \\'blows\\', \\'unreal\\', \\'horrors\\', \\'aloud\\', \\'coda\\', \\'wack\\', \\'ignite\\', \\'kindling\\', \\'swift\\', \\'visions\\', \\'predicted\\', \\'visionary\\', \\'outcast\\', \\'predicts\\', \\'prophesy\\', \\'climate\\', \\'thinkers\\', \\'enlightenment\\', \\'warnings\\', \\'citizens\\', \\'prophecies\\', \\'basing\\', \\'societies\\', \\'apocalypse\\', \\'utopian\\', \\'revolutionary\\', \\'ruled\\', \\'threat\\', \\'democracy\\', \\'liberties\\', \\'rebellion\\', \\'anti\\', \\'individuality\\', \\'enforced\\', \\'authority\\', \\'communist\\', \\'striking\\', \\'selected\\', \\'notion\\', \\'eighth\\', \\'enforcement\\', \\'laws\\', \\'humankind\\', \\'propaganda\\', \\'sci\\', \\'hung\\', \\'collective\\', \\'fears\\', \\'mentality\\', \\'consequence\\', \\'insightful\\', \\'openly\\', \\'prophetic\\', \\'commit\\', \\'radical\\', \\'forces\\', \\'fed\\', \\'sixth\\', \\'independent\\', \\'beings\\', \\'no-brainer\\', \\'owning\\', \\'norm\\', \\'startling\\', \\'substantial\\', \\'questioning\\', \\'libraries\\', \\'contest\\', \\'senior\\', \\'dialogues\\', \\'reasoning\\', \\'strengths\\', \\'gloom\\', \\'motivation\\', \\'existential\\', \\'accomplish\\', \\'committed\\', \\'discourse\\', \\'ans\\', \\'growth\\', \\'imprisoned\\', \\'contribution\\', \\'discussing\\', \\'shockingly\\', \\'bounds\\', \\'reflects\\', \\'advocate\\', \\'hellish\\', \\'parallel\\', \\'emphasis\\', \\'largely\\', \\'deadly\\', \\'shared\\', \\'repeatedly\\', \\'beliefs\\', \\'concerning\\', \\'strugles\\', \\'peters\\', \\'dreaded\\', \\'meeting\\', \\'thoughtful\\', \\'suicide\\', \\'taut\\', \\'scholar\\', \\'happiness\\', \\'challenges\\', \\'reflected\\', \\'whereas\\', \\'attacks\\', \\'tolerance\\', \\'dramatically\\', \\'masses\\', \\'behold\\', \\'sincere\\', \\'intrest\\', \\'introductions\\', \\'argue\\', \\'reports\\', \\'lent\\', \\'focuses\\', \\'centered\\', \\'justify\\', \\'destroying\\', \\'realise\\', \\'agrees\\', \\'richly\\', \\'succeeds\\', \\'reckon\\', \\'existence\\', \\'chosen\\', \\'memorial\\', \\'symbol\\', \\'bleak\\', \\'intersting\\', \\'shocking\\', \\'involves\\', \\'leaders\\', \\'dynamic\\', \\'brainwashed\\', \\'county\\', \\'mentions\\', \\'challenged\\', \\'teen\\', \\'centers\\', \\'dedicated\\', \\'objects\\', \\'established\\', \\'assignment\\', \\'remembering\\', \\'realization\\', \\'lobbyist\\', \\'targets\\', \\'thy\\', \\'nervous\\', \\'indictment\\', \\'wise\\', \\'employed\\', \\'dreamed\\', \\'instance\\', \\'identify\\', \\'comparing\\', \\'requirement\\', \\'touches\\', \\'applies\\', \\'leisure\\', \\'birds\\', \\'penchant\\', \\'parental\\', \\'pedestrian\\', \\'comparisons\\', \\'resolved\\', \\'wonders\\', \\'realm\\', \\'reached\\', \\'regain\\', \\'constructed\\', \\'forms\\', \\'categories\\', \\'sophomore\\', \\'necessarily\\', \\'carries\\', \\'rely\\', \\'affected\\', \\'eloquent\\', \\'tied\\', \\'agony\\', \\'documents\\', \\'strive\\', \\'thsi\\', \\'summed\\', \\'surrounding\\', \\'indicating\\', \\'array\\', \\'stealing\\', \\'innocence\\', \\'awake\\', \\'winning\\', \\'meanings\\', \\'teenagers\\', \\'hunting\\', \\'storytelling\\', \\'filed\\', \\'lucid\\', \\'usage\\', \\'chaos\\', \\'revolves\\', \\'begining\\', \\'fable\\', \\'lies\\', \\'exception\\', \\'demise\\', \\'unanswered\\', \\'cram\\', \\'analyze\\', \\'junior\\', \\'detracts\\', \\'captain\\', \\'tru\\', \\'occurred\\', \\'throws\\', \\'cell\\', \\'inventive\\', \\'adopted\\', \\'survival\\', \\'complexity\\', \\'whilst\\', \\'brillant\\', \\'comprehend\\', \\'intent\\', \\'font\\', \\'adjectives\\', \\'popularity\\', \\'minority\\', \\'brutally\\', \\'contrary\\', \\'spouse\\', \\'ignores\\', \\'far-fetched\\', \\'visited\\', \\'dare\\', \\'randomly\\', \\'idiotic\\', \\'pivotal\\', \\'hackneyed\\', \\'novella\\', \\'ebook\\', \\'lengthy\\', \\'occasions\\', \\'ties\\', \\'topical\\', \\'correctness\\', \\'passionate\\', \\'needing\\', \\'ranting\\', \\'dives\\', \\'hardback\\', \\'decline\\', \\'redundant\\', \\'containing\\', \\'ludicrous\\', \\'doubts\\', \\'neighbor\\', \\'examination\\', \\'dreadfully\\', \\'upsetting\\', \\'offense\\', \\'crammed\\', \\'civilization\\', \\'thier\\', \\'outraged\\', \\'genesis\\', \\'rediculous\\', \\'twisting\\', \\'womens\\', \\'insane\\', \\'primal\\', \\'wierd\\', \\'trudge\\', \\'tender\\', \\'appetizing\\', \\'continuing\\', \\'eerily\\', \\'unimportant\\', \\'censored\\', \\'abandon\\', \\'horrified\\', \\'saves\\', \\'struggled\\', \\'confuse\\', \\'miserably\\', \\'unconvincing\\', \\'devoid\\', \\'civilized\\', \\'desperate\\', \\'nonexistent\\', \\'refuses\\', \\'cancer\\', \\'disappears\\', \\'degrees\\', \\'horrific\\', \\'backwards\\', \\'characterizations\\', \\'mildly\\', \\'tempting\\', \\'stumbles\\', \\'cunning\\', \\'chore\\', \\'deterioration\\', \\'choas\\', \\'drags\\', \\'banal\\', \\'mildred\\', \\'poses\\', \\'unfinished\\', \\'vapid\\', \\'headed\\', \\'aka\\', \\'monotonous\\', \\'typically\\', \\'qualities\\', \\'clarisse\\', \\'hiding\\', \\'inappropriate\\', \\'forgets\\', \\'hounds\\', \\'mild\\', \\'basement\\', \\'flowing\\', \\'pacing\\', \\'midnight\\', \\'hatred\\', \\'asks\\', \\'eaten\\', \\'satirical\\', \\'chopped\\', \\'murderer\\', \\'dinner\\', \\'hurts\\', \\'butt\\', \\'longest\\', \\'dissatisfaction\\', \\'cheated\\', \\'hadnt\\', \\'failing\\', \\'nine\\', \\'rain\\', \\'delete\\', \\'slap\\', \\'sparks\\', \\'reset\\', \\'recommened\\', \\'ideal\\', \\'toss\\', \\'nearby\\', \\'proved\\', \\'goodie\\', \\'conditions\\', \\'accepted\\', \\'robbed\\', \\'gladly\\', \\'excellently\\', \\'bestseller\\', \\'gently\\', \\'placing\\', \\'sends\\', \\'throwing\\', \\'clicked\\', \\'boards\\', \\'permanently\\', \\'builds\\', \\'elegant\\', \\'butter\\', \\'rebels\\', \\'wiped\\', \\'scale\\', \\'brittle\\', \\'spice\\', \\'stir\\', \\'spoon\\', \\'stirring\\', \\'upsets\\', \\'wrap\\', \\'warmth\\', \\'knife\\', \\'rotten\\', \\'mechanical\\', \\'dose\\', \\'matches\\', \\'screwed\\', \\'fireplace\\', \\'walked\\', \\'absorb\\', \\'sky\\', \\'longing\\', \\'opener\\', \\'stepped\\', \\'rolled\\', \\'beneath\\', \\'rude\\', \\'floats\\', \\'rip-off\\', \\'rainy\\', \\'acutally\\', \\'yea\\', \\'oldie\\', \\'t.v\\', \\'automated\\', \\'mounting\\', \\'monitored\\', \\'stabbed\\', \\'disgusted\\', \\'lauded\\', \\'disguised\\', \\'drunk\\', \\'crave\\', \\'marvelous\\', \\'bombed\\', \\'priceless\\', \\'horrendous\\', \\'fights\\', \\'fighters\\', \\'rendered\\']',\n",
       " 44: \"\\n            Topic index: 44\\n            Topic name: Totalitarianism and Surveillance\\n            Topic description: The common topic of the given words is dystopian literature. \\n\\nAspects and sub-topics of the topic include:\\n1. Totalitarianism and oppression: governments, ruling, surveillance, telescreens, propaganda, doublethink.\\n2. Loss of privacy and individuality: privacy, surveillance, freedoms, liberty, individuality.\\n3. Grim and nightmarish society: grim, harrowing, nightmarish, oppressive, dystopian.\\n4. Satire and critique of political systems: satire, totalitarianism, regime, rebel, communist.\\n5. Predictions and relevance to current society: prediction, relevance, prophetic, warnings, nowadays.\\n            Topic topwords: ['governments', 'satire', 'prediction', 'totalitarianism', 'privacy', 'predictions', 'ruling', 'utopia', 'surveillance', 'rebel', 'dystopian', 'regime', 'measures', 'grim', 'telescreens', 'newspeak', 'propaganda', 'doublethink', 'oppression', 'freedoms', 'tyranny', 'fears', 'liberty', 'ruled', 'ideals', 'predicting', 'relevance', 'thought-provoking', 'ministry', 'portrays', 'harrowing', 'nightmarish', 'soviet', 'communist', 'utopian', 'citizens', 'oppressive', 'conspiracy', 'prescient', 'populace', 'politically', 'individuality', 'technological', 'subjected', 'increasingly', 'evolving', 'representative', 'viewed', 'controversial', 'stood', 'institutions', 'regarded', 'climate', 'prophetic', 'similarities', 'criminals', 'excess', 'cautionary', 'assignment', 'corporate', 'masses', 'histories', 'prophet', 'leadership', 'behaviour', 'approaches', 'identify', 'forbidden', 'characteristics', 'applicable', 'unfair', 'existence', 'minded', 'relates', 'rightfully', 'disgruntled', 'emphasize', 'blindly', 'proles', 'disturbed', 'cynical', 'spectrum', 'lies', 'tortured', 'documents', 'eliminated', 'e.g.', 'critique', 'occur', 'trace', 'meanings', 'realizes', 'inevitably', 'criticize', 'afterword', 'accuracy', 'therein', 'lesser', 'betrays', 'vs.', 'disappear', 'wierd', 'eerily', 'fetched', 'resemble', 'erased', 'proved', 'conditions', 'sells', 'havent', 'displays', 'taped', 'riveted', 'drones', 'continuously', 'wound', 'springs', 'spoilers', 'destruction', 'orwells', 'paranoia', 'overbearing', 'rebellion', 'controlling', 'democracy', 'threat', 'eighth', 'nations', 'dictator', 'individualism', 'invasion', 'socialism', 'predict', 'visionary', 'foresight', 'dangers', 'societal', 'powers', 'enlightenment', 'prophecy', 'representation', 'possibilities', 'uprising', 'predicted', 'warnings', 'nowadays', 'organizations', 'retrospect', 'forces', 'manipulate', 'prevalent', 'thinker', 'resistance', 'circa', 'rethink', 'existing', 'differently', 'predicts', 'controller', 'influenced', 'attacks', 'politicians', 'discuss', 'capable', 'exposes', 'cautious', 'collect', 'technologies', 'consistent', 'awareness', 'neighbors', 'fifty', 'inevitable', 'shortly', 'counter', 'dictate', 'ethical', 'fundamentally', 'significance', 'conservative', 'founding', 'worker', 'greedy', 'truths', 'efficiency', 'rightly', 'capitalist', 'withstood', 'compelled', 'tactics', 'persons', 'capitalism', 'victory', 'approve', 'neighborhood', 'wrenching', 'agencies', 'impacts', 'essays', 'commit', 'treatise', 'depends', 'convincing', 'reflects', 'committed', 'credits', 'revealed', 'must-read', 'outcome', 'objectives', 'greed', 'intensity', 'brainwashed', 'dominated', 'brilliantly', 'questioning', 'jail', 'losses', 'leaders', 'legal', 'ponderous', 'altered', 'reminder', 'protection', 'reflect', 'stopping', 'tremendously', 'shifting', 'belief', 'conveys', 'transition', 'mastery', 'progressed', 'greatness', 'contributes', 'develops', 'targeted', 'hack', 'shockingly', 'mindset', 'behaviors', 'interactions', 'hopeful', 'forthright', 'imprisoned', 'clarity', 'encourage', 'souls', 'promote', 'responses', 'experiencing', 'purely', 'facing', 'poignant', 'achieved', 'trading', 'philosophical', 'worlds', 'beliefs', 'artificial', 'scholars', 'thouroughly', 'indictment', 'tracing', 'eloquently', 'psychology', 'romanticism', 'thoughtful', 'achievement', 'appreciation', 'beneficial', 'expanded', 'familiarity', 'suicide', 'discourage', 'resonates', 'judge', 'proudly', 'approached', 'dismal', 'rewarded', 'mans', 'protected', 'beings', 'identity', 'handbook', 'profoundly', 'distinction', 'lessen', 'surrounding', 'wondered', 'shocking', 'focusing', 'assault', 'enduring', 'detached', 'glory', 'reached', 'reaction', 'streets', 'bodies', 'pursue', 'legitimate', 'polemic', 'honesty', 'bleak', 'ignorance', 'conclude', 'established', 'fruition', 'cerebral', 'sought', 'parallels', 're-reading', 'intentions', 'medicines', 'parallel', 'enthralled', 'quirks', 'deepest', 'referenced', 'broad', 'solely', 'instincts', 'guard', 'reaching', 'follow-up', 'provisions', 'allusions', 'referred', 'illustrate', 'cult', 'eliminate', 'proving', 'entirety', 'challenged', 'staggering', 'virtue', 'explores', 'connections', 'elaborate', 'subjective', 'immature', 'loads', 'spark', 'searches', 'meaningful', 'highschool', 'demonstrate', 'abrupt', 'ostensibly', 'i.e.', 'ignored', 'logical', 'defined', 'criticizing', 'reactions', 'maintains', 'preachy', 'survived', 'teen', 'collectors', 'tenacity', 'demonstrates', 'happily', 'regard', 'cried', 'depicts', 'dating', 'countless', 'contrast', 'dreaming', 'depiction', 'nerve', 'expression', 'grader', 'followers', 'insinuations', 'positions', 'absorbed', 'childs', 'proverbial', 'non-fiction', 'monologues', 'crimes', 'intrested', 'bookshelf', 'vital', 'arent', 'afterthought', 'blog', 'mundane', 'wonders', 'writings', 'weaved', 'curiosity', 'oral', 'intent', 'sentiments', 'questionable', 'attacking', 'phases', 'spelling', 'odds', 'anger', 'pioneer', 'collecting', 'shortest', 'implemented', 'striving', 'exposed', 'neighbor', 'unlikely', 'accents', 'writting', 'grandmother', 'drew', 'drives', 'depressed', 'exiled', 'intrigued', 'hunger', 'outer', 'contradiction', 'parks', 'enchanting', 'reviewing', 'begining', 'escaped', 'understatement', 'trapped', 'vividly', 'shouting', 'pynchon', 'adjectives', 'thier', 'obtain', 'spell', 'wrought', 'maps', 'restaurants', 'he/she', 'comprehend', 'enthralling', 'hating', 'topical', 'far-fetched', 'portray', 'thesis', 'consistently', 'redundant', 'buzz', 'posters', 'continuing', 'correcting', 'unfortunate', 'nothingness', 'passage', 'inclined', 'varies', 'delves', 'sustained', 'warns', 'ruining', 'depicted', 'page-turner', 'versus', 'non-existent', 'dialouge', 'suspence', 'affairs', 'wordy', 'centuries', 'decade', 'bores', 'turgid', 'relax', 'bogged', 'unecessary', 'dreary', 'devoid', 'resulted', 'flowers', 'shadow', 'side-by-side', 'nightmares', 'edits', 'skimmed', 'danish', 'frightful', 'horrific', 'refuses', 'sorely', 'remainder', 'pitfalls', 'unexpected', 'unabridged', 'partly', 'skewed', 'faceless']\"}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tm.pprompt(\"What are the differences and similarities of topic 39 and topic 44?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since the topics seem sufficiently similar, we will combine them "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT wants to the call the function:  {\n",
      "  \"name\": \"combine_topics\",\n",
      "  \"arguments\": \"{\\n  \\\"topic_idx_lis\\\": [39, 44],\\n  \\\"inplace\\\": true\\n}\"\n",
      "}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epochs completed: 100%| ██████████ 100/100 [00:02]\n",
      "Computing word-topic matrix: 100%|██████████| 1/1 [00:00<00:00,  2.53it/s]\n",
      "Epochs completed: 100%| ██████████ 30/30 [00:09]\n",
      "Epochs completed: 100%| ██████████ 100/100 [00:02]\n",
      "100%|██████████| 1/1 [00:10<00:00, 10.79s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topics 39 and 44 have been merged into a new topic called \"Dystopian Governments\". This topic encompasses aspects of dystopian literature, including the portrayal of oppressive ruling regimes in governments and totalitarianism, characters making thought-provoking predictions, outlawed firefighters in a dystopian society, constant monitoring and loss of privacy through surveillance, and the use of satire and irony to critique societal and political issues.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[Topic 0: Music Genres,\n",
       " Topic 1: Sci-fi TV series.,\n",
       " Topic 2: Movie genres,\n",
       " Topic 3: Paranormal Phenomena,\n",
       " Topic 4: Earbuds and Headsets,\n",
       " Topic 5: Literary Satire,\n",
       " Topic 6: Gluten-free Recipes,\n",
       " Topic 7: Inflatable Air Mattresses.,\n",
       " Topic 8: Crime Trials,\n",
       " Topic 9: Printer Troubleshooting,\n",
       " Topic 10: Hiking Shoes,\n",
       " Topic 11: Shapewear and Body Contouring,\n",
       " Topic 12: Dance Instruction.,\n",
       " Topic 13: Prenatal Education and Support,\n",
       " Topic 14: Electronic Gadgets,\n",
       " Topic 15: Gaming Experience.,\n",
       " Topic 16: MP3 Player Issues,\n",
       " Topic 17: Camera Accessories,\n",
       " Topic 18: Electronics Accessories,\n",
       " Topic 19: Amazon Product Reviews,\n",
       " Topic 20: Historical Anthropology,\n",
       " Topic 21: Wireless Router Performance,\n",
       " Topic 22: Technical Compatibility Issues,\n",
       " Topic 23: Puritan society and guilt.,\n",
       " Topic 24: Sci-fi Space Exploration,\n",
       " Topic 25: Beauty and skincare,\n",
       " Topic 26: Sexual pleasure products,\n",
       " Topic 27: Home Safety Devices,\n",
       " Topic 28: Online Shopping,\n",
       " Topic 29: Product Issues,\n",
       " Topic 30: Test Preparation,\n",
       " Topic 31: Programming Documentation,\n",
       " Topic 32: Hardware Tools,\n",
       " Topic 33: Animal Attacks,\n",
       " Topic 34: Educational Toys,\n",
       " Topic 35: Product Quality,\n",
       " Topic 36: Kitchenware,\n",
       " Topic 37: Supernatural Witches,\n",
       " Topic 38: Horror comics,\n",
       " Topic 39: Emotional Turmoil,\n",
       " Topic 40: Literature and Poetry,\n",
       " Topic 41: Political and Economic Critique,\n",
       " Topic 42: Negative Book Reviews,\n",
       " Topic 43: Poetry Analysis,\n",
       " Topic 44: Poor Formatting,\n",
       " Topic 45: Language Learning Resources,\n",
       " Topic 46: Books about Various Themes.,\n",
       " Topic 47: Home Improvement,\n",
       " Topic 48: Religious Literature,\n",
       " Topic 49: book publishing,\n",
       " Topic 50: Dystopian Governments]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tm.pprompt(\"please merge topic 39 and topic 44. Do this inplace\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm_sem",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}