source: src/main/java/uva/projectai/y2018/jasparon/Qlearner.java@ 84

package uva.projectai.y2018.jasparon;

import java.io.FileInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.HashMap;

import genius.core.Bid;
import genius.core.actions.Action;
import genius.core.actions.EndNegotiation;
import genius.core.boaframework.BOAagentBilateral;
import genius.core.events.MultipartyNegoActionEvent;
import genius.core.events.NegotiationEvent;
import negotiator.boaframework.acceptanceconditions.other.AC_Next;
import negotiator.boaframework.omstrategy.BestBid;
import negotiator.boaframework.opponentmodel.PerfectModel;

import uva.projectai.y2018.jasparon.QlearningStrategy;

@SuppressWarnings("serial")
public class Qlearner extends BOAagentBilateral implements RLBOA {

        QlearningStrategy offeringStrategy;
        String state;

        @Override
        public void agentSetup() {
                
                System.out.print(this.instanceIdentifier());
                HashMap<String, Double> params = new HashMap<String, Double>();

                // Use perfect opponent model to decrease noise in the environment
                opponentModel = new PerfectModel();
                opponentModel.init(negotiationSession, params);

                // Load existing qTable
                @SuppressWarnings("unchecked")
                HashMap<Integer, ArrayList<Double>> qTable = (HashMap<Integer, ArrayList<Double>>) readObjectFromFile("qTables/" + this.instanceIdentifier());

                // Initialize offeringStrategy (is a RL-component)
                switch (this.getStrategyParameters().getValueAsString("strategy")) {
                        case "QlearningStrategy":
                                offeringStrategy = new QlearningStrategy(negotiationSession, opponentModel);
                                break;
                        case "PriorBeliefQlearningStrategy":
                                offeringStrategy = new PriorBeliefQlearningStrategy(negotiationSession, opponentModel);
                                break;
                        case "QLambdaStrategy":
                                offeringStrategy = new QLambdaStrategy(negotiationSession, opponentModel);
                                break;
                }
                offeringStrategy.setHyperparameters(this.getStrategyParameters());
                offeringStrategy.initQtable(qTable);

                // Accept if the incoming offer is higher than what you would offer yourself
                acceptConditions = new AC_Next(negotiationSession, offeringStrategy, 1, 0);

                // Opponent model strategy always selects best bid it has available
                omStrategy = new BestBid();
                omStrategy.init(negotiationSession, opponentModel, params);
                setDecoupledComponents(acceptConditions, offeringStrategy, opponentModel, omStrategy);
        }

        @Override
        public String getName() {
                return "Q-learner";
        }

        public State getStateRepresentation(MultipartyNegoActionEvent negoEvent) {
                Bid oppLastBid = negotiationSession.getOpponentBidHistory().getLastBid();
                Bid myLastBid = negotiationSession.getOwnBidHistory().getLastBid();
                Bid agreement = negoEvent.getAgreement();
                Action currentAction = negoEvent.getAction();

                if (agreement != null || currentAction.getClass() == EndNegotiation.class) {
                        return State.TERMINAL;
                }

                int myBin;
                if (myLastBid != null) {
                        double myBidUtil = this.getUtility(myLastBid);
                        myBin = this.getBinIndex(myBidUtil);
                }
                else {
                        myBin = Integer.MIN_VALUE;
                }

                int oppBin;
                if (oppLastBid != null) {
                        double oppBidUtil = this.getUtility(oppLastBid);
                        oppBin = this.getBinIndex(oppBidUtil);
                }
                else {
                        oppBin = Integer.MIN_VALUE;
                }

                double time = negotiationSession.getTime();

                State state = new State(myBin, oppBin, this.getTimeBinIndex(time));

                return state;
        }

        private int getBinIndex(double util) {
                int n_bins = offeringStrategy.getNBins();
                return (int) Math.floor(util * n_bins);
        }

        private int getTimeBinIndex(double time) {
                // TODO: Remove magic number
                return (int) Math.floor(time * 4);
        }

        @Override
        public void notifyChange(NegotiationEvent data) {
                if (data instanceof MultipartyNegoActionEvent) {
                        // Get relevant information from negotiation event
                        MultipartyNegoActionEvent negoEvent = (MultipartyNegoActionEvent) data;

                        // Observe state
                        State newState = this.getStateRepresentation(negoEvent);

                        double reward = this.getReward(negoEvent);
                        boolean myTurn = negoEvent.getAction().getAgent() == this.getAgentID();

                        if (newState.isTerminalState() || !myTurn) {
                                this.observeEnvironment(reward, newState);
                        }
                }
        }

        @Override
        public double getReward(MultipartyNegoActionEvent negoEvent) {
                double reward = 0.0;
                Bid agreement = negoEvent.getAgreement();

                if (agreement != null) {
                        reward =  this.getUtility(agreement);
                }

                return reward;
        }

        @Override
        public void observeEnvironment(double reward, State newState) {
                this.offeringStrategy.observeEnvironment(reward, newState);

                if (newState.isTerminalState()) {
                        this.writeObjectToFile(this.offeringStrategy.getQTable());
                }
        }

        public void writeObjectToFile(Object serObj) {
                String DIRECTORY = "qTables/";
                String filepath = DIRECTORY + this.instanceIdentifier();
                
                File directory = new File(DIRECTORY);
                if (!directory.exists()) {
                        directory.mkdir();
                }
                
                try {
                        FileOutputStream fileOut = new FileOutputStream(filepath);
                        ObjectOutputStream objectOut = new ObjectOutputStream(fileOut);
                        objectOut.writeObject(serObj);
                        objectOut.close();
                        System.out.println("The Object  was succesfully written to a file");

                } catch (Exception ex) {
                        ex.printStackTrace();
                }
        }

        public Object readObjectFromFile(String filepath) {
                Object obj = null;

                try {
                        FileInputStream fileIn = new FileInputStream(filepath);
                        ObjectInputStream objectIn = new ObjectInputStream(fileIn);
                        obj = objectIn.readObject();
                        objectIn.close();
                        System.out.println("Succesfully read object");
                }
                catch (Exception ex) {
                        if (ex instanceof FileNotFoundException ) {
                                System.out.println("qTable file does not exist. A new file will be created.");
                        }
                        else {
                                ex.printStackTrace();
                        }
                }

                return obj;
        }

        public String instanceIdentifier() {
                return String.format("Qlearner-%s", this.getStrategyParameters().toString());
        }
}