From 4fbbb51575bd7eadbfdd02ffdc433cb15cf8f476 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Bj=C3=B8rn=20Rustad?= Date: Fri, 3 Nov 2017 20:18:39 +0100 Subject: [PATCH] Initial commit --- glmnet.r | 17 +++ testing123.ipynb | 345 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 362 insertions(+) create mode 100644 glmnet.r create mode 100644 testing123.ipynb diff --git a/glmnet.r b/glmnet.r new file mode 100644 index 0000000..ce827b5 --- /dev/null +++ b/glmnet.r @@ -0,0 +1,17 @@ +library(glmnet) + +df_train <- read.csv("numerai_training_data.csv", header=TRUE) +df_tourn <- read.csv("numerai_tournament_data.csv", header=TRUE) + +xnam <- paste("feature", 1:50, sep="") + +train_mat <- as.matrix(df_train[xnam]) +tourn_mat <- as.matrix(df_tourn[xnam]) + +cv_fit <- cv.glmnet(train_mat, df_train$target, family="binomial") +best.lambda <- cv_fit$lambda.1se +fit <- glmnet(train_mat, df_train$target, family="binomial", lambda=best.lambda) +pred <- predict.glmnet(newx=tourn_mat, object=fit, s=best.lambda, type="response", family="binomial") +pred <- exp(pred) / (exp(pred) + 1) +ple <- paste(df_tourn$id, pred) +write.csv(ple, file="predictions.csv", row.names=FALSE, quote=FALSE) \ No newline at end of file diff --git a/testing123.ipynb b/testing123.ipynb new file mode 100644 index 0000000..0412c8a --- /dev/null +++ b/testing123.ipynb @@ -0,0 +1,345 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import sklearn\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n", + "from sklearn.metrics import log_loss\n", + "from sklearn.linear_model import LogisticRegression" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "featlist = [\"feature{}\".format(x) for x in [3,4,6,10,11,12,16,17,18,19,21,22,24,25,26,31,38,40,41]]\n", + "featlist = [\"feature{}\".format(x) for x in [5,12,19,23,32,33,35,49,50]]\n", + "df = pd.read_csv(\"numerai_training_data.csv\")\n", + "X_full = df[featlist]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "train, test = train_test_split(df, test_size=0.2, random_state=1337)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "X = train[featlist]\n", + "y = train['target']" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "X_test = test[featlist]\n", + "y_test = test['target']" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [], + "source": [ + "lr = LogisticRegression(C=0.1)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LogisticRegression(C=0.1, class_weight=None, dual=False, fit_intercept=True,\n", + " intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n", + " penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n", + " verbose=0, warm_start=False)" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lr.fit(X, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred = lr.predict_proba(X_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loss: 0.6918104092706736\n" + ] + } + ], + "source": [ + "loss = log_loss(y_test, y_pred)\n", + "print(\"Loss: {}\".format(loss))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "test_df = pd.read_csv(\"numerai_tournament_data.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "X_tournament = test_df.filter(regex='feature')" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [], + "source": [ + "out = pd.DataFrame({ \"id\": test_df['id'], \"probability\": [y[0] for y in y_tournament]})" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [], + "source": [ + "out.to_csv(\"upload.csv\", columns=[\"id\", \"probability\"], index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "from keras.models import Sequential\n", + "from keras.layers import Dense, Dropout, Conv1D, Reshape\n", + "from keras.optimizers import SGD" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/burk/dev/numerai/venv/lib/python3.5/site-packages/keras/models.py:848: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n", + " warnings.warn('The `nb_epoch` argument in `fit` '\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train on 428570 samples, validate on 107143 samples\n", + "Epoch 1/2\n", + "428570/428570 [==============================] - 21s - loss: 0.6967 - val_loss: 0.6934\n", + "Epoch 2/2\n", + "428570/428570 [==============================] - 22s - loss: 0.6934 - val_loss: 0.6933\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# logistic regression with L1 and L2 regularization\n", + "from keras.regularizers import l1_l2\n", + "\n", + "reg = l1_l2(l1=0.01, l2=0.01)\n", + "\n", + "model = Sequential()\n", + "model.add(Dense(1, activation='sigmoid', kernel_regularizer=reg, input_dim=X.shape[1]))\n", + "model.compile(optimizer='rmsprop', loss='binary_crossentropy')\n", + "model.fit(np.array(X), np.array(y), nb_epoch=2, validation_data=(np.array(X_test), np.array(y_test)))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/burk/dev/numerai/venv/lib/python3.5/site-packages/keras/models.py:848: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n", + " warnings.warn('The `nb_epoch` argument in `fit` '\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train on 428570 samples, validate on 107143 samples\n", + "Epoch 1/3\n", + "428570/428570 [==============================] - 20s - loss: 0.6941 - val_loss: 0.6933\n", + "Epoch 2/3\n", + "428570/428570 [==============================] - 19s - loss: 0.6927 - val_loss: 0.6930\n", + "Epoch 3/3\n", + "428570/428570 [==============================] - 19s - loss: 0.6923 - val_loss: 0.6926\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# logistic regression with L1 and L2 regularization\n", + "from keras.regularizers import l1_l2\n", + "\n", + "reg = l1_l2(l1=0.0001, l2=0.0001)\n", + "\n", + "model = Sequential()\n", + "model.add(Dense(1, activation='sigmoid', kernel_regularizer=reg, input_dim=X.shape[1]))\n", + "#model.add(Dropout(0.5))\n", + "#model.add(Dense(1, activation='sigmoid', kernel_regularizer=reg))\n", + "model.compile(optimizer=SGD(lr=0.01), loss='binary_crossentropy')\n", + "model.fit(np.array(X), np.array(y), nb_epoch=3, validation_data=(np.array(X_test), np.array(y_test)))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "345568/348689 [============================>.] - ETA: 0s" + ] + } + ], + "source": [ + "y_tournament = model.predict_proba(np.array(X_tournament[featlist]))\n", + "out = pd.DataFrame({ \"id\": test_df['id'], \"probability\": [y[0] for y in y_tournament]})\n", + "out.to_csv(\"upload.csv\", columns=[\"id\", \"probability\"], index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "104960/107143 [============================>.] - ETA: 0s[[ 0.54190403]\n", + " [ 0.49770615]\n", + " [ 0.50981015]\n", + " ..., \n", + " [ 0.50999701]\n", + " [ 0.5024662 ]\n", + " [ 0.51308435]]\n" + ] + } + ], + "source": [ + "print(model.predict_proba(np.array(X_test)))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} -- 2.47.3