FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Diabetes Prediction using python by anmolbhatt0 · Pull Request #829 · seeditsolution/pythonprogram · GitHub

Repository navigation

Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension No extension  (1) All 1 file type selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
97 changes: 97 additions & 0 deletions Diabetes-prediction
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
# -*- coding: utf-8 -*-

!apt-get install openjdk-8-jdk-headless -qq > /dev/null
!pip install pyspark==2.4.4

"""# Environment Path"""

import os
os.environ['JAVA_HOME'] = '/usr/lib/jvm/java-8-openjdk-amd64'

"""# Run a SparkSession"""

from pyspark.sql import SparkSession

spark = SparkSession.builder.appName("spark").getOrCreate()

"""# Clone Diabetes Dataset"""

! git clone https://github.com/education454/diabetes_dataset

! ls diabetes_dataset

df = spark.read.csv('/content/diabetes_dataset/diabetes.csv',header=True,inferSchema=True)

df.show()

df.printSchema()

df.groupby('Outcome').count().show()

df.describe().show()

"""# Cleaning Data"""

for col in df.columns:
print(col+":",df[df[col].isNull()].count())

def count_zeros():
columns_list =['Glucose','BloodPressure','SkinThickness','Insulin','BMI']
for i in columns_list:
print(i+":",df[df[i]==0].count())

count_zeros()

from pyspark.sql.functions import *
for i in df.columns[1:6]:
data = df.agg({i:'mean'}).first()[0]
print("Mean value for {} is {}".format(i,int(data)))
df = df.withColumn(i,when(df[i]==0,int(data)).otherwise(df[i]))

df.show()

"""# Correlation"""

for col in df.columns:
print("correlation to outcome for {} is {}".format(col,df.stat.corr('Outcome',col)))

"""# Feature Selection"""

from pyspark.ml.feature import VectorAssembler
assembler = VectorAssembler(inputCols=['Pregnancies','Glucose','BloodPressure','SkinThickness','Insulin','BMI','DiabetesPedigreeFunction','Age'],outputCol='features')
output_data = assembler.transform(df)

output_data.printSchema()

output_data.show()

"""# Build & Train Model"""

from pyspark.ml.classification import LogisticRegression
final_data = output_data.select('features','Outcome')

final_data.printSchema()

train , test = final_data.randomSplit([0.7,0.3])
models = LogisticRegression(labelCol='Outcome')
model = models.fit(train)

summary = model.summary

summary.predictions.describe().show()

"""# Evaluation & Test Model"""

from pyspark.ml.evaluation import BinaryClassificationEvaluator
predictions = model.evaluate(test)

predictions.predictions.show(20)

evaluator = BinaryClassificationEvaluator(rawPredictionCol='rawPrediction', labelCol='Outcome')
evaluator.evaluate(model.transform(test))

model.save("model")

from pyspark.ml.classification import LogisticRegressionModel
model = LogisticRegressionModel.load('model')


Back | FazBrowse Home | New Git URL