+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "import pandas as pd\n",
+ "import sklearn\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
+ "from sklearn.metrics import log_loss\n",
+ "from sklearn.linear_model import LogisticRegression"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "featlist = [\"feature{}\".format(x) for x in [3,4,6,10,11,12,16,17,18,19,21,22,24,25,26,31,38,40,41]]\n",
+ "featlist = [\"feature{}\".format(x) for x in [5,12,19,23,32,33,35,49,50]]\n",
+ "df = pd.read_csv(\"numerai_training_data.csv\")\n",
+ "X_full = df[featlist]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train, test = train_test_split(df, test_size=0.2, random_state=1337)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "X = train[featlist]\n",
+ "y = train['target']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "X_test = test[featlist]\n",
+ "y_test = test['target']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = LogisticRegression(C=0.1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 56,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "LogisticRegression(C=0.1, class_weight=None, dual=False, fit_intercept=True,\n",
+ " intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
+ " penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
+ " verbose=0, warm_start=False)"
+ ]
+ },
+ "execution_count": 56,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "lr.fit(X, y)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "y_pred = lr.predict_proba(X_test)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Loss: 0.6918104092706736\n"
+ ]
+ }
+ ],
+ "source": [
+ "loss = log_loss(y_test, y_pred)\n",
+ "print(\"Loss: {}\".format(loss))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_df = pd.read_csv(\"numerai_tournament_data.csv\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "X_tournament = test_df.filter(regex='feature')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 69,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "out = pd.DataFrame({ \"id\": test_df['id'], \"probability\": [y[0] for y in y_tournament]})"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 73,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "out.to_csv(\"upload.csv\", columns=[\"id\", \"probability\"], index=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using TensorFlow backend.\n"
+ ]
+ }
+ ],
+ "source": [
+ "from keras.models import Sequential\n",
+ "from keras.layers import Dense, Dropout, Conv1D, Reshape\n",
+ "from keras.optimizers import SGD"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/burk/dev/numerai/venv/lib/python3.5/site-packages/keras/models.py:848: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
+ " warnings.warn('The `nb_epoch` argument in `fit` '\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Train on 428570 samples, validate on 107143 samples\n",
+ "Epoch 1/2\n",
+ "428570/428570 [==============================] - 21s - loss: 0.6967 - val_loss: 0.6934\n",
+ "Epoch 2/2\n",
+ "428570/428570 [==============================] - 22s - loss: 0.6934 - val_loss: 0.6933\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "<keras.callbacks.History at 0x7f528dd78668>"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# logistic regression with L1 and L2 regularization\n",
+ "from keras.regularizers import l1_l2\n",
+ "\n",
+ "reg = l1_l2(l1=0.01, l2=0.01)\n",
+ "\n",
+ "model = Sequential()\n",
+ "model.add(Dense(1, activation='sigmoid', kernel_regularizer=reg, input_dim=X.shape[1]))\n",
+ "model.compile(optimizer='rmsprop', loss='binary_crossentropy')\n",
+ "model.fit(np.array(X), np.array(y), nb_epoch=2, validation_data=(np.array(X_test), np.array(y_test)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/burk/dev/numerai/venv/lib/python3.5/site-packages/keras/models.py:848: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
+ " warnings.warn('The `nb_epoch` argument in `fit` '\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Train on 428570 samples, validate on 107143 samples\n",
+ "Epoch 1/3\n",
+ "428570/428570 [==============================] - 20s - loss: 0.6941 - val_loss: 0.6933\n",
+ "Epoch 2/3\n",
+ "428570/428570 [==============================] - 19s - loss: 0.6927 - val_loss: 0.6930\n",
+ "Epoch 3/3\n",
+ "428570/428570 [==============================] - 19s - loss: 0.6923 - val_loss: 0.6926\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "<keras.callbacks.History at 0x7f52655f9550>"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# logistic regression with L1 and L2 regularization\n",
+ "from keras.regularizers import l1_l2\n",
+ "\n",
+ "reg = l1_l2(l1=0.0001, l2=0.0001)\n",
+ "\n",
+ "model = Sequential()\n",
+ "model.add(Dense(1, activation='sigmoid', kernel_regularizer=reg, input_dim=X.shape[1]))\n",
+ "#model.add(Dropout(0.5))\n",
+ "#model.add(Dense(1, activation='sigmoid', kernel_regularizer=reg))\n",
+ "model.compile(optimizer=SGD(lr=0.01), loss='binary_crossentropy')\n",
+ "model.fit(np.array(X), np.array(y), nb_epoch=3, validation_data=(np.array(X_test), np.array(y_test)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "345568/348689 [============================>.] - ETA: 0s"
+ ]
+ }
+ ],
+ "source": [
+ "y_tournament = model.predict_proba(np.array(X_tournament[featlist]))\n",
+ "out = pd.DataFrame({ \"id\": test_df['id'], \"probability\": [y[0] for y in y_tournament]})\n",
+ "out.to_csv(\"upload.csv\", columns=[\"id\", \"probability\"], index=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "104960/107143 [============================>.] - ETA: 0s[[ 0.54190403]\n",
+ " [ 0.49770615]\n",
+ " [ 0.50981015]\n",
+ " ..., \n",
+ " [ 0.50999701]\n",
+ " [ 0.5024662 ]\n",
+ " [ 0.51308435]]\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(model.predict_proba(np.array(X_test)))"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.5.2"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
projects / numerai / commitdiff